Merge pull request #1392 from jtchen-study/master

DataX 通道能力更新（MaxCompute、Hologres、Tdengine、OSS）、安全漏洞更新、通用打包更新等

NOTICE

@@ -0,0 +1,39 @@
+========================================================
+DataX 是阿里云 DataWorks数据集成 的开源版本，在阿里巴巴集团内被广泛使用的离线数据同步工具/平台。DataX 实现了包括 MySQL、Oracle、OceanBase、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、Hologres、DRDS 等各种异构数据源之间高效的数据同步功能。
+
+DataX is an open source offline data synchronization tool / platform widely used in Alibaba group and other companies. DataX implements efficient data synchronization between heterogeneous data sources including mysql, Oracle, oceanbase, sqlserver, postgre, HDFS, hive, ads, HBase, tablestore (OTS), maxcompute (ODPs), hologres, DRDS, etc.
+
+Copyright 1999-2022 Alibaba Group Holding Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+===================================================================
+文级别引用，按许可证
+This product contains various third-party components under other open source licenses.
+This section summarizes those components and their licenses. 
+GNU Lesser General Public License
+--------------------------------------
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/CliQuery.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/Connection4TSDB.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/DataPoint4TSDB.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/DumpSeries.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/OpenTSDBConnection.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/conn/OpenTSDBDump.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/Constant.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/Key.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/OpenTSDBReader.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/OpenTSDBReaderErrorCode.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/util/HttpUtils.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/util/TSDBUtils.java
+opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/util/TimeUtils.java
+===================================================================

README.md

@@ -62,7 +62,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
 |              | Elasticsearch       |         |     √     |[写](https://github.com/alibaba/DataX/blob/master/elasticsearchwriter/doc/elasticsearchwriter.md)|
 | 时间序列数据库 | OpenTSDB | √ |  |[读](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md)|
 |  | TSDB | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/tsdbreader/doc/tsdbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md)|
-|  | TDengine | √ | √ |[读](https://github.com/taosdata/DataX/blob/master/tdenginereader/doc/tdenginereader.md) 、[写](https://github.com/taosdata/DataX/blob/master/tdenginewriter/doc/tdenginewriter-CN.md)|
+|  | TDengine | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/tdenginereader/doc/tdenginereader-CN.md) 、[写](https://github.com/alibaba/DataX/blob/master/tdenginewriter/doc/tdenginewriter-CN.md)|
 
 # 阿里云DataWorks数据集成
 

adbpgwriter/src/main/doc/adbpgwriter.md

@@ -65,9 +65,9 @@ COPY命令将数据写入ADB PG数据库中。
             "writer": {
               "name": "adbpgwriter",
               "parameter": {
-                  "username": "username",
-                  "password": "password",
-                  "host": "host",
+                  "username": "",
+                  "password": "",
+                  "host": "127.0.0.1",
                   "port": "1234",
                   "database": "database",
                   "schema": "schema",

common/pom.xml

@@ -61,6 +61,14 @@
     </dependencies>
 
     <build>
+        <resources>
+          <resource>
+            <directory>src/main/java</directory>
+            <includes>
+              <include>**/*.properties</include>
+            </includes>
+          </resource>
+        </resources>
         <plugins>
             <plugin>
                 <artifactId>maven-compiler-plugin</artifactId>

common/src/main/java/com/alibaba/datax/common/element/BoolColumn.java

@@ -92,7 +92,13 @@ public Date asDate() {
 		throw DataXException.asDataXException(
 				CommonErrorCode.CONVERT_NOT_SUPPORT, "Bool类型不能转为Date .");
 	}
-
+
+	@Override
+	public Date asDate(String dateFormat) {
+		throw DataXException.asDataXException(
+				CommonErrorCode.CONVERT_NOT_SUPPORT, "Bool类型不能转为Date .");
+	}
+
 	@Override
 	public byte[] asBytes() {
 		throw DataXException.asDataXException(

common/src/main/java/com/alibaba/datax/common/element/BytesColumn.java

@@ -75,6 +75,12 @@ public Date asDate() {
 		throw DataXException.asDataXException(
 				CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Date .");
 	}
+
+	@Override
+	public Date asDate(String dateFormat) {
+		throw DataXException.asDataXException(
+				CommonErrorCode.CONVERT_NOT_SUPPORT, "Bytes类型不能转为Date .");
+	}
 
 	@Override
 	public Boolean asBoolean() {

common/src/main/java/com/alibaba/datax/common/element/Column.java

@@ -55,6 +55,8 @@ protected void setByteSize(int byteSize) {
 	public abstract String asString();
 
 	public abstract Date asDate();
+
+	public abstract Date asDate(String dateFormat);
 
 	public abstract byte[] asBytes();
 

common/src/main/java/com/alibaba/datax/common/element/ColumnCast.java

@@ -22,6 +22,11 @@ public static Date string2Date(final StringColumn column)
 			throws ParseException {
 		return StringCast.asDate(column);
 	}
+
+	public static Date string2Date(final StringColumn column, String dateFormat)
+			throws ParseException {
+		return StringCast.asDate(column, dateFormat);
+	}
 
 	public static byte[] string2Bytes(final StringColumn column)
 			throws UnsupportedEncodingException {
@@ -115,6 +120,16 @@ static Date asDate(final StringColumn column) throws ParseException {
 		}
 		throw e;
 	}
+
+	static Date asDate(final StringColumn column, String dateFormat) throws ParseException {
+		ParseException e;
+		try {
+			return FastDateFormat.getInstance(dateFormat, StringCast.timeZoner).parse(column.asString());
+		} catch (ParseException ignored) {
+			e = ignored;
+		}
+		throw e;
+	}
 
 	static byte[] asBytes(final StringColumn column)
 			throws UnsupportedEncodingException {

common/src/main/java/com/alibaba/datax/common/element/DateColumn.java

@@ -89,6 +89,11 @@ public Date asDate() {
 
 		return new Date((Long)this.getRawData());
 	}
+
+	@Override
+	public Date asDate(String dateFormat) {
+		return asDate();
+	}
 
 	@Override
 	public byte[] asBytes() {

common/src/main/java/com/alibaba/datax/common/element/DoubleColumn.java

@@ -132,6 +132,12 @@ public Date asDate() {
 		throw DataXException.asDataXException(
 				CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Date类型 .");
 	}
+
+	@Override
+	public Date asDate(String dateFormat) {
+		throw DataXException.asDataXException(
+				CommonErrorCode.CONVERT_NOT_SUPPORT, "Double类型无法转为Date类型 .");
+	}
 
 	@Override
 	public byte[] asBytes() {

common/src/main/java/com/alibaba/datax/common/element/LongColumn.java

@@ -125,6 +125,11 @@ public Date asDate() {
 		}
 		return new Date(this.asLong());
 	}
+
+	@Override
+	public Date asDate(String dateFormat) {
+		return this.asDate();
+	}
 
 	@Override
 	public byte[] asBytes() {

common/src/main/java/com/alibaba/datax/common/element/Record.java

@@ -1,5 +1,7 @@
 package com.alibaba.datax.common.element;
 
+import java.util.Map;
+
 /**
  * Created by jingxing on 14-8-24.
  */
@@ -20,4 +22,8 @@ public interface Record {
 
 	public int getMemorySize();
 
+	public void setMeta(Map<String, String> meta);
+
+	public Map<String, String> getMeta();
+
 }

common/src/main/java/com/alibaba/datax/common/element/StringColumn.java

@@ -149,6 +149,16 @@ public Date asDate() {
 					String.format("String[\"%s\"]不能转为Date .", this.asString()));
 		}
 	}
+
+	@Override
+	public Date asDate(String dateFormat) {
+		try {
+			return ColumnCast.string2Date(this, dateFormat);
+		} catch (Exception e) {
+			throw DataXException.asDataXException(CommonErrorCode.CONVERT_NOT_SUPPORT,
+					String.format("String[\"%s\"]不能转为Date .", this.asString()));
+		}
+	}
 
 	@Override
 	public byte[] asBytes() {

common/src/main/java/com/alibaba/datax/common/exception/DataXException.java

@@ -16,6 +16,10 @@ public DataXException(ErrorCode errorCode, String errorMessage) {
         this.errorCode = errorCode;
     }
 
+    public DataXException(String errorMessage) {
+        super(errorMessage);
+    }
+
     private DataXException(ErrorCode errorCode, String errorMessage, Throwable cause) {
         super(errorCode.toString() + " - " + getMessage(errorMessage) + " - " + getMessage(cause), cause);
 
@@ -26,6 +30,10 @@ public static DataXException asDataXException(ErrorCode errorCode, String messag
         return new DataXException(errorCode, message);
     }
 
+    public static DataXException asDataXException(String message) {
+        return new DataXException(message);
+    }
+
     public static DataXException asDataXException(ErrorCode errorCode, String message, Throwable cause) {
         if (cause instanceof DataXException) {
             return (DataXException) cause;

common/src/main/java/com/alibaba/datax/common/plugin/AbstractPlugin.java

@@ -3,6 +3,8 @@
 import com.alibaba.datax.common.base.BaseObject;
 import com.alibaba.datax.common.util.Configuration;
 
+import java.util.List;
+
 public abstract class AbstractPlugin extends BaseObject implements Pluginable {
 	//作业的config
     private Configuration pluginJobConf;
@@ -15,6 +17,8 @@ public abstract class AbstractPlugin extends BaseObject implements Pluginable {
 
     private String peerPluginName;
 
+    private List<Configuration> readerPluginSplitConf;
+
     @Override
 	public String getPluginName() {
 		assert null != this.pluginConf;
@@ -84,4 +88,12 @@ public void preHandler(Configuration jobConfiguration){
     public void postHandler(Configuration jobConfiguration){
 
     }
+
+    public List<Configuration> getReaderPluginSplitConf(){
+        return this.readerPluginSplitConf;
+    }
+
+    public void setReaderPluginSplitConf(List<Configuration> readerPluginSplitConf){
+        this.readerPluginSplitConf = readerPluginSplitConf;
+    }
 }

common/src/main/java/com/alibaba/datax/common/util/ConfigurationUtil.java

@@ -0,0 +1,37 @@
+package com.alibaba.datax.common.util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class ConfigurationUtil {
+    private static final List<String> SENSITIVE_KEYS = Arrays.asList("password", "accessKey", "securityToken",
+        "AccessKeyId", "AccessKeySecert", "AccessKeySecret", "clientPassword");
+
+    public static Configuration filterSensitive(Configuration origin) {
+        // shell 任务configuration metric 可能为null。
+        if (origin == null) {
+            return origin;
+        }
+        // 确保不影响入参的对象
+        Configuration configuration = origin.clone();
+        Set<String> keys = configuration.getKeys();
+        for (final String key : keys) {
+            boolean isSensitive = false;
+            for (String sensitiveKey : SENSITIVE_KEYS) {
+                if (StringUtils.endsWithIgnoreCase(key, sensitiveKey)) {
+                    isSensitive = true;
+                    break;
+                }
+            }
+
+            if (isSensitive && configuration.get(key) instanceof String) {
+                configuration.set(key, configuration.getString(key).replaceAll(".", "*"));
+            }
+
+        }
+        return configuration;
+    }
+}