Add docker internal server check script (#2092)

* add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add logger port and modify dolphinscheduler_env to dolphinscheduler_env.sh
apache · Jun 29, 2020 · 58046b4 · 58046b4
1 parent fdc9e00
commit 58046b4
Show file tree

Hide file tree

Showing 9 changed files with 108 additions and 30 deletions.
diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile
@@ -63,19 +63,22 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
 ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
 
 #7. add configuration and modify permissions and set soft links
+ADD ./checkpoint.sh /root/checkpoint.sh
 ADD ./startup-init-conf.sh /root/startup-init-conf.sh
 ADD ./startup.sh /root/startup.sh
 ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
-ADD ./conf/dolphinscheduler/env/dolphinscheduler_env /opt/dolphinscheduler/conf/env/
-RUN chmod +x /root/startup-init-conf.sh && \
+ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
+RUN chmod +x /root/checkpoint.sh && \
+    chmod +x /root/startup-init-conf.sh && \
     chmod +x /root/startup.sh && \
-    chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \
+    chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     chmod +x /opt/dolphinscheduler/script/*.sh && \
     chmod +x /opt/dolphinscheduler/bin/*.sh && \
     chmod +x /opt/zookeeper/bin/*.sh && \
+    dos2unix /root/checkpoint.sh && \
     dos2unix /root/startup-init-conf.sh && \
     dos2unix /root/startup.sh && \
-    dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \
+    dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     dos2unix /opt/dolphinscheduler/script/*.sh && \
     dos2unix /opt/dolphinscheduler/bin/*.sh && \
     dos2unix /opt/zookeeper/bin/*.sh && \
@@ -87,6 +90,6 @@ RUN chmod +x /root/startup-init-conf.sh && \
 RUN rm -rf /var/cache/apk/*
 
 #9. expose port
-EXPOSE 2181 2888 3888 5432 12345 8888
+EXPOSE 2181 2888 3888 5432 12345 50051 8888
 
 ENTRYPOINT ["/root/startup.sh"]
diff --git a/dockerfile/README.md b/dockerfile/README.md
@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
 #### You can start a dolphinscheduler instance
 ```
 $ docker run -dit --name dolphinscheduler \ 
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \
+-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -25,13 +25,13 @@ The default postgres user `root`, postgres password `root` and database `dolphin
 
 The default zookeeper is created in the `startup.sh`.
 
-#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`**
+#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`**
 
 You can specify **existing postgres service**. Example:
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 -p 8888:8888 \
 dolphinscheduler all
@@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -56,7 +56,7 @@ You can start a standalone dolphinscheduler server.
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 dolphinscheduler master-server
 ```
@@ -66,7 +66,7 @@ dolphinscheduler master-server
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 dolphinscheduler worker-server
 ```
@@ -75,7 +75,7 @@ dolphinscheduler worker-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 -p 12345:12345 \
 dolphinscheduler api-server
@@ -85,7 +85,7 @@ dolphinscheduler api-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 dolphinscheduler alert-server
 ```
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
 dolphinscheduler frontend
 ```
 
-**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
+**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
 
 ## How to build a docker image
 
@@ -140,14 +140,36 @@ This environment variable sets the port for PostgreSQL. The default value is `54
 
 This environment variable sets the username for PostgreSQL. The default value is `root`.
 
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
 **`POSTGRESQL_PASSWORD`**
 
 This environment variable sets the password for PostgreSQL. The default value is `root`.
 
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`POSTGRESQL_DATABASE`**
+
+This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
 **`DOLPHINSCHEDULER_ENV_PATH`**
 
 This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`.
 
+**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
+
+User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler`
+
+**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`**
+
+Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download`
+
+**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`**
+
+Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec`
+
 **`TASK_QUEUE`**
 
 This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`.

diff --git a/dockerfile/README_zh_CN.md b/dockerfile/README_zh_CN.md
@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
 #### 你可以运行一个dolphinscheduler实例
 ```
 $ docker run -dit --name dolphinscheduler \ 
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \
+-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -31,7 +31,7 @@ dolphinscheduler all
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 -p 8888:8888 \
 dolphinscheduler all
@@ -42,7 +42,7 @@ dolphinscheduler all
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -p 8888:8888 \
 dolphinscheduler all
 ```
@@ -56,7 +56,7 @@ dolphinscheduler all
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 dolphinscheduler master-server
 ```
@@ -66,7 +66,7 @@ dolphinscheduler master-server
 ```
 $ docker run -dit --name dolphinscheduler \
 -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 dolphinscheduler worker-server
 ```
@@ -75,7 +75,7 @@ dolphinscheduler worker-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 -p 12345:12345 \
 dolphinscheduler api-server
@@ -85,7 +85,7 @@ dolphinscheduler api-server
 
 ```
 $ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
+-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
 -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
 dolphinscheduler alert-server
 ```
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
 dolphinscheduler frontend
 ```
 
-**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM`。
+**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。
 
 ## 如何构建一个docker镜像
 
@@ -140,14 +140,36 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
 
 配置`PostgreSQL`的`USERNAME`， 默认值 `root`。
 
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
 **`POSTGRESQL_PASSWORD`**
 
 配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。
 
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`POSTGRESQL_DATABASE`**
+
+配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
 **`DOLPHINSCHEDULER_ENV_PATH`**
 
 任务执行时的环境变量配置文件， 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。
 
+**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
+
+用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler`。
+
+**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`**
+
+用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/download`。
+
+**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`**
+
+任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/exec`。
+
 **`TASK_QUEUE`**
 
 配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名， 默认值 `zookeeper`。

diff --git a/dockerfile/checkpoint.sh b/dockerfile/checkpoint.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then
+    echo "[ERROR] $1 process not exits."
+    exit 1
+else
+    echo "[INFO] $1 process exits."
+    exit 0
+fi
diff --git a/dockerfile/conf/dolphinscheduler/application.properties.tpl b/dockerfile/conf/dolphinscheduler/application.properties.tpl
@@ -19,7 +19,7 @@
 spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
 # postgre
 spring.datasource.driver-class-name=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8
+spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
 # mysql
 #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
 #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8

diff --git a/dockerfile/conf/dolphinscheduler/common.properties.tpl b/dockerfile/conf/dolphinscheduler/common.properties.tpl
@@ -38,6 +38,12 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH}
 resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties
 # is development state? default "false"
 development.state=true
+# user data directory path, self configuration, please make sure the directory exists and have read write permissions
+data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH}
+# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
+data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH}
+# process execute directory. self configuration, please make sure the directory exists and have read write permissions
+process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH}
 
 # resource upload startup type : HDFS,S3,NONE
 res.upload.startup.type=NONE
@@ -49,12 +55,6 @@ res.upload.startup.type=NONE
 hdfs.root.user=hdfs
 # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
 data.store2hdfs.basepath=/dolphinscheduler
-# user data directory path, self configuration, please make sure the directory exists and have read write permissions
-data.basedir.path=/tmp/dolphinscheduler
-# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
-data.download.basedir.path=/tmp/dolphinscheduler/download
-# process execute directory. self configuration, please make sure the directory exists and have read write permissions
-process.exec.basepath=/tmp/dolphinscheduler/exec
 # whether kerberos starts
 hadoop.security.authentication.startup.state=false
 # java.security.krb5.conf path

diff --git a/...dolphinscheduler/env/dolphinscheduler_env → ...phinscheduler/env/dolphinscheduler_env.sh b/...dolphinscheduler/env/dolphinscheduler_env → ...phinscheduler/env/dolphinscheduler_env.sh
diff --git a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl
@@ -22,7 +22,7 @@
 org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
 # postgre
 org.quartz.dataSource.myDs.driver = org.postgresql.Driver
-org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8
+org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
 org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME}
 org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD}
 org.quartz.scheduler.instanceName = DolphinScheduler

diff --git a/dockerfile/startup-init-conf.sh b/dockerfile/startup-init-conf.sh
@@ -28,11 +28,15 @@ export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"}
 export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"}
 export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"}
 export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"}
+export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"}
 
 #============================================================================
 # System
 #============================================================================
 export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
+export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
+export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"}
+export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"}
 
 #============================================================================
 # Zookeeper