Skip to content

Commit

Permalink
Merge pull request #1 from apache/dev
Browse files Browse the repository at this point in the history
commits
  • Loading branch information
yinancx authored Sep 22, 2020
2 parents 6e7cce0 + db663a1 commit a1ee3b5
Show file tree
Hide file tree
Showing 231 changed files with 10,663 additions and 4,619 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/ci_e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ jobs:
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
sudo dpkg -i google-chrome*.deb
sudo apt-get install -f -y
wget -N https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_linux64.zip
google-chrome -version
googleVersion=$(curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE)
wget -N https://chromedriver.storage.googleapis.com/${googleVersion}/chromedriver_linux64.zip
unzip chromedriver_linux64.zip
sudo mv -f chromedriver /usr/local/share/chromedriver
sudo ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ If remote branch has a new branch `DEV-1.0`, you need to synchronize this branch

```
git checkout -b dev-1.0 upstream/dev-1.0
git push --set-upstream origin dev1.0
git push --set-upstream origin dev-1.0
```

## Create your feature branch
Expand Down
31 changes: 16 additions & 15 deletions docker/build/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,13 @@ ENV DEBIAN_FRONTEND noninteractive
#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:
#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
RUN apk update && \
apk --update add --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \
apk add --update procps && \
apk add --update --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \
apk add --update --no-cache procps && \
openrc boot && \
pip install kazoo

#2. install jdk
RUN apk add openjdk8
RUN apk add --update --no-cache openjdk8
ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
ENV PATH $JAVA_HOME/bin:$PATH

Expand All @@ -43,19 +43,20 @@ RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler

#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed
RUN apk add postgresql postgresql-contrib
RUN apk add --update --no-cache postgresql postgresql-contrib

#5. modify nginx
RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
rm -rf /etc/nginx/conf.d/*
ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
COPY ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d

#6. add configuration and modify permissions and set soft links
ADD ./checkpoint.sh /root/checkpoint.sh
ADD ./startup-init-conf.sh /root/startup-init-conf.sh
ADD ./startup.sh /root/startup.sh
ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
COPY ./checkpoint.sh /root/checkpoint.sh
COPY ./startup-init-conf.sh /root/startup-init-conf.sh
COPY ./startup.sh /root/startup.sh
COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
COPY conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
RUN chmod +x /root/checkpoint.sh && \
chmod +x /root/startup-init-conf.sh && \
chmod +x /root/startup.sh && \
Expand All @@ -70,12 +71,12 @@ RUN chmod +x /root/checkpoint.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \
rm -rf /bin/sh && \
ln -s /bin/bash /bin/sh && \
mkdir -p /tmp/xls

#7. remove apk index cache and disable coredup for sudo
RUN rm -rf /var/cache/apk/* && \
mkdir -p /tmp/xls && \
#7. remove apk index cache and disable coredup for sudo
rm -rf /var/cache/apk/* && \
echo "Set disable_coredump false" >> /etc/sudo.conf


#8. expose port
EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888

Expand Down
2 changes: 2 additions & 0 deletions docker/build/conf/zookeeper/zoo.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,5 @@ clientPort=2181
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
#Four Letter Words commands:stat,ruok,conf,isro
4lw.commands.whitelist=*
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.dolphinscheduler.alert.utils;

import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.*;

import org.apache.dolphinscheduler.plugin.model.AlertData;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
Expand All @@ -29,37 +30,39 @@
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Enterprise WeChat utils
*/
public class EnterpriseWeChatUtils {

public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class);

public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID);
public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS);
private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID);

private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET);

private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL);
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL
.replaceAll("\\{corpId\\}", ENTERPRISE_WE_CHAT_CORP_ID)
.replaceAll("\\{secret\\}", ENTERPRISE_WE_CHAT_SECRET);

.replaceAll("\\{corpId}", ENTERPRISE_WE_CHAT_CORP_ID)
.replaceAll("\\{secret}", ENTERPRISE_WE_CHAT_SECRET);
private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL);

private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG);

private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG);

public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID);

public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS);
private static final String agentIdRegExp = "\\{agentId}";
private static final String msgRegExp = "\\{msg}";
private static final String userRegExp = "\\{toUser}";

/**
* get Enterprise WeChat is enable
Expand Down Expand Up @@ -116,70 +119,70 @@ public static String getToken() throws IOException {
*
* @param toParty the toParty
* @param agentId the agentId
* @param msg the msg
* @param msg the msg
* @return Enterprise WeChat send message
*/
public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", toParty)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", toParty)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
}

/**
* make team multi Enterprise WeChat message
*
* @param toParty the toParty
* @param agentId the agentId
* @param msg the msg
* @param msg the msg
* @return Enterprise WeChat send message
*/
public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|");
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", listParty)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", listParty)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
}

/**
* make team single user message
*
* @param toUser the toUser
* @param toUser the toUser
* @param agentId the agentId
* @param msg the msg
* @param msg the msg
* @return Enterprise WeChat send message
*/
public static String makeUserSendMsg(String toUser, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", toUser)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser}", toUser)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
}

/**
* make team multi user message
*
* @param toUser the toUser
* @param toUser the toUser
* @param agentId the agentId
* @param msg the msg
* @param msg the msg
* @return Enterprise WeChat send message
*/
public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|");
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", listUser)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(userRegExp, listUser)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
}

/**
* send Enterprise WeChat
*
* @param charset the charset
* @param data the data
* @param token the token
* @param data the data
* @param token the token
* @return Enterprise WeChat resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""}
* @throws IOException the IOException
*/
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token\\}", token);
String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token}", token);

CloseableHttpClient httpClient = HttpClients.createDefault();
try {
Expand All @@ -205,7 +208,7 @@ public static String sendEnterpriseWeChat(String charset, String data, String to
/**
* convert table to markdown style
*
* @param title the title
* @param title the title
* @param content the content
* @return markdown table content
*/
Expand All @@ -215,13 +218,13 @@ public static String markdownTable(String title, String content) {

if (null != mapItemsList) {
for (LinkedHashMap mapItems : mapItemsList) {
Set<Map.Entry<String, String>> entries = mapItems.entrySet();
Iterator<Map.Entry<String, String>> iterator = entries.iterator();
Set<Map.Entry<String, Object>> entries = mapItems.entrySet();
Iterator<Map.Entry<String, Object>> iterator = entries.iterator();
StringBuilder t = new StringBuilder(String.format("`%s`%s", title, Constants.MARKDOWN_ENTER));

while (iterator.hasNext()) {

Map.Entry<String, String> entry = iterator.next();
Map.Entry<String, Object> entry = iterator.next();
t.append(Constants.MARKDOWN_QUOTE);
t.append(entry.getKey()).append(":").append(entry.getValue());
t.append(Constants.MARKDOWN_ENTER);
Expand All @@ -235,30 +238,31 @@ public static String markdownTable(String title, String content) {
/**
* convert text to markdown style
*
* @param title the title
* @param title the title
* @param content the content
* @return markdown text
*/
public static String markdownText(String title, String content) {
if (StringUtils.isNotEmpty(content)) {
List<String> list;
try {
list = JSONUtils.toList(content, String.class);
} catch (Exception e) {
logger.error("json format exception", e);
return null;
}
List<LinkedHashMap> mapItemsList = JSONUtils.toList(content, LinkedHashMap.class);
if (null != mapItemsList) {
StringBuilder contents = new StringBuilder(100);
contents.append(String.format("`%s`%n", title));
for (LinkedHashMap mapItems : mapItemsList) {

Set<Map.Entry<String, Object>> entries = mapItems.entrySet();
Iterator<Map.Entry<String, Object>> iterator = entries.iterator();
while (iterator.hasNext()) {
Map.Entry<String, Object> entry = iterator.next();
contents.append(Constants.MARKDOWN_QUOTE);
contents.append(entry.getKey()).append(":").append(entry.getValue());
contents.append(Constants.MARKDOWN_ENTER);
}

StringBuilder contents = new StringBuilder(100);
contents.append(String.format("`%s`%n", title));
for (String str : list) {
contents.append(Constants.MARKDOWN_QUOTE);
contents.append(str);
contents.append(Constants.MARKDOWN_ENTER);
}
return contents.toString();
}

return contents.toString();

}
return null;
}
Expand All @@ -278,4 +282,5 @@ public static String markdownByAlert(AlertData alert) {
return result;

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public class MailUtils {

public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE);

public static final String SSL_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE);
public static final Boolean SSL_ENABLE = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE);

public static final String SSL_TRUST = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST);

Expand Down Expand Up @@ -213,6 +213,7 @@ private static MimeMessage getMimeMessage(Collection<String> receivers) throws M

/**
* get session
*
* @return the new Session
*/
private static Session getSession() {
Expand All @@ -222,8 +223,10 @@ private static Session getSession() {
props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE);
props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL);
props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE);
props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, SSL_ENABLE);
props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST);
if (SSL_ENABLE) {
props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, "true");
props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST);
}

Authenticator auth = new Authenticator() {
@Override
Expand Down Expand Up @@ -345,5 +348,5 @@ private static void handleException(Collection<String> receivers, Map<String, Ob
retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed," + e.toString());
}


}
Loading

0 comments on commit a1ee3b5

Please sign in to comment.