diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml
index eb25da493..110eea394 100644
--- a/amazon-redshift-plugin/pom.xml
+++ b/amazon-redshift-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Amazon Redshift plugin
diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml
index ce6fc8052..4f4ac4907 100644
--- a/aurora-mysql-plugin/pom.xml
+++ b/aurora-mysql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Aurora DB MySQL plugin
diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml
index 2fe68b64e..421f557cc 100644
--- a/aurora-postgresql-plugin/pom.xml
+++ b/aurora-postgresql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Aurora DB PostgreSQL plugin
diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml
index 706502fc9..60212b67d 100644
--- a/cloudsql-mysql-plugin/pom.xml
+++ b/cloudsql-mysql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
CloudSQL MySQL plugin
diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml
index 66d8a05f0..bbbda9df1 100644
--- a/cloudsql-postgresql-plugin/pom.xml
+++ b/cloudsql-postgresql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
CloudSQL PostgreSQL plugin
diff --git a/database-commons/pom.xml b/database-commons/pom.xml
index bd5de4128..a59453390 100644
--- a/database-commons/pom.xml
+++ b/database-commons/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Database Commons
diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml
index 3a600eaaf..f1db30f12 100644
--- a/db2-plugin/pom.xml
+++ b/db2-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
IBM DB2 plugin
diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml
index 3db66ca8f..e9dea0b93 100644
--- a/generic-database-plugin/pom.xml
+++ b/generic-database-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Generic database plugin
diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml
index af9ce1797..ef450a352 100644
--- a/generic-db-argument-setter/pom.xml
+++ b/generic-db-argument-setter/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Generic database argument setter plugin
diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml
index a37b89019..0537b9dbf 100644
--- a/mariadb-plugin/pom.xml
+++ b/mariadb-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Maria DB plugin
diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml
index 6c0e4304e..bcd20bf3e 100644
--- a/memsql-plugin/pom.xml
+++ b/memsql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Memsql plugin
diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml
index bda544b24..348f8d5c0 100644
--- a/mssql-plugin/pom.xml
+++ b/mssql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Microsoft SQL Server plugin
diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml
index 238ff2170..23a5aca38 100644
--- a/mysql-plugin/pom.xml
+++ b/mysql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Mysql plugin
diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml
index 415450aa2..9ebe52134 100644
--- a/netezza-plugin/pom.xml
+++ b/netezza-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Netezza plugin
diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml
index 7074373c4..1db95d3c1 100644
--- a/oracle-plugin/pom.xml
+++ b/oracle-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
Oracle plugin
diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java
index 3d2f7399a..16371d5c1 100644
--- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java
+++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java
@@ -112,7 +112,8 @@ protected DBConnectorPath getDBConnectorPath(String path) {
@Override
protected SchemaReader getSchemaReader(String sessionID) {
- return new OracleSourceSchemaReader(sessionID);
+ return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(),
+ config.getTreatPrecisionlessNumAsDeci());
}
@Override
diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java
index c3ce051e5..cbc1e5ed2 100644
--- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java
+++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java
@@ -22,8 +22,6 @@
import io.cdap.plugin.db.TransactionIsolationLevel;
import io.cdap.plugin.db.connector.AbstractDBSpecificConnectorConfig;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Properties;
import javax.annotation.Nullable;
@@ -43,12 +41,14 @@ public OracleConnectorConfig(String host, int port, String user, String password
public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName,
String connectionArguments, String connectionType, String database) {
- this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null);
+ this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null,
+ null);
}
public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName,
String connectionArguments, String connectionType, String database,
- String role, Boolean useSSL) {
+ String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp,
+ @Nullable Boolean treatPrecisionlessNumAsDeci) {
this.host = host;
this.port = port;
@@ -60,6 +60,8 @@ public OracleConnectorConfig(String host, int port, String user, String password
this.database = database;
this.role = role;
this.useSSL = useSSL;
+ this.treatAsOldTimestamp = treatAsOldTimestamp;
+ this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci;
}
@Override
@@ -86,6 +88,16 @@ public String getConnectionString() {
@Nullable
public Boolean useSSL;
+ @Name(OracleConstants.TREAT_AS_OLD_TIMESTAMP)
+ @Description("A hidden field to handle timestamp as CDAP's timestamp micros or string as per old behavior.")
+ @Nullable
+ public Boolean treatAsOldTimestamp;
+
+ @Name(OracleConstants.TREAT_PRECISIONLESSNUM_AS_DECI)
+ @Description("A hidden field to handle precision less number as CDAP's decimal per old behavior.")
+ @Nullable
+ public Boolean treatPrecisionlessNumAsDeci;
+
@Override
protected int getDefaultPort() {
return 1521;
@@ -108,6 +120,14 @@ public Boolean getSSlMode() {
return useSSL != null && useSSL;
}
+ public Boolean getTreatAsOldTimestamp() {
+ return Boolean.TRUE.equals(treatAsOldTimestamp);
+ }
+
+ public Boolean getTreatPrecisionlessNumAsDeci() {
+ return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci);
+ }
+
@Override
public Properties getConnectionArgumentsProperties() {
Properties prop = super.getConnectionArgumentsProperties();
diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java
index dc38f80ac..cbd411175 100644
--- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java
+++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java
@@ -43,6 +43,8 @@ private OracleConstants() {
public static final String TNS_CONNECTION_TYPE = "tns";
public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel";
public static final String USE_SSL = "useSSL";
+ public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp";
+ public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci";
/**
* Constructs the Oracle connection string based on the provided connection type, host, port, and database.
diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java
index 6df62e63e..583fd686f 100644
--- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java
+++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java
@@ -63,7 +63,12 @@ protected String createConnectionString() {
@Override
protected SchemaReader getSchemaReader() {
- return new OracleSourceSchemaReader();
+ // PLUGIN-1893 : Based on field/properties from Oracle source and Oracle connection we will pass the flag to control
+ // handle schema to make it backward compatible.
+ boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp();
+ boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci();
+
+ return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci);
}
@Override
@@ -117,9 +122,11 @@ public OracleSourceConfig(String host, int port, String user, String password, S
String connectionArguments, String connectionType, String database, String role,
int defaultBatchValue, int defaultRowPrefetch,
String importQuery, Integer numSplits, int fetchSize,
- String boundingQuery, String splitBy, Boolean useSSL) {
+ String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp,
+ Boolean treatPrecisionlessNumAsDeci) {
this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments,
- connectionType, database, role, useSSL);
+ connectionType, database, role, useSSL, treatAsOldTimestamp,
+ treatPrecisionlessNumAsDeci);
this.defaultBatchValue = defaultBatchValue;
this.defaultRowPrefetch = defaultRowPrefetch;
this.fetchSize = fetchSize;
diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java
index 7d35f9bc7..dd17d2e84 100644
--- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java
+++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java
@@ -26,6 +26,7 @@
import java.sql.SQLException;
import java.sql.Types;
import java.util.Set;
+import javax.annotation.Nullable;
/**
* Oracle Source schema reader.
@@ -65,14 +66,17 @@ public class OracleSourceSchemaReader extends CommonSchemaReader {
);
private final String sessionID;
+ private final Boolean isTimestampOldBehavior;
+ private final Boolean isPrecisionlessNumAsDecimal;
public OracleSourceSchemaReader() {
- this(null);
+ this(null, false, false);
}
-
- public OracleSourceSchemaReader(String sessionID) {
- super();
+ public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior,
+ boolean isPrecisionlessNumAsDecimal) {
this.sessionID = sessionID;
+ this.isTimestampOldBehavior = isTimestampOldBehavior;
+ this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal;
}
@Override
@@ -81,10 +85,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti
switch (sqlType) {
case TIMESTAMP_TZ:
- return Schema.of(Schema.LogicalType.TIMESTAMP_MICROS);
- case Types.TIMESTAMP:
+ return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS);
case TIMESTAMP_LTZ:
- return Schema.of(Schema.LogicalType.DATETIME);
+ return isTimestampOldBehavior ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)
+ : Schema.of(Schema.LogicalType.DATETIME);
+ case Types.TIMESTAMP:
+ return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME);
case BINARY_FLOAT:
return Schema.of(Schema.Type.FLOAT);
case BINARY_DOUBLE:
@@ -107,12 +113,24 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti
// For a Number type without specified precision and scale, precision will be 0 and scale will be -127
if (precision == 0) {
// reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832
- LOG.warn(String.format("Field '%s' is a %s type without precision and scale, "
- + "converting into STRING type to avoid any precision loss.",
- metadata.getColumnName(index),
- metadata.getColumnTypeName(index),
- metadata.getColumnName(index)));
- return Schema.of(Schema.Type.STRING);
+ if (isPrecisionlessNumAsDecimal) {
+ precision = 38;
+ scale = 0;
+ LOG.warn(String.format("%s type with undefined precision and scale is detected, "
+ + "there may be a precision loss while running the pipeline. "
+ + "Please define an output precision and scale for field '%s' to avoid "
+ + "precision loss.",
+ metadata.getColumnTypeName(index),
+ metadata.getColumnName(index)));
+ return Schema.decimalOf(precision, scale);
+ } else {
+ LOG.warn(String.format("Field '%s' is a %s type without precision and scale, "
+ + "converting into STRING type to avoid any precision loss.",
+ metadata.getColumnName(index),
+ metadata.getColumnTypeName(index),
+ metadata.getColumnName(index)));
+ return Schema.of(Schema.Type.STRING);
+ }
}
return Schema.decimalOf(precision, scale);
}
diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json
index 5eca20cc4..404262fb2 100644
--- a/oracle-plugin/widgets/Oracle-batchsource.json
+++ b/oracle-plugin/widgets/Oracle-batchsource.json
@@ -120,6 +120,44 @@
]
}
},
+ {
+ "widget-type": "hidden",
+ "label": "Treat as old timestamp",
+ "name": "treatAsOldTimestamp",
+ "widget-attributes": {
+ "layout": "inline",
+ "default": "false",
+ "options": [
+ {
+ "id": "true",
+ "label": "true"
+ },
+ {
+ "id": "false",
+ "label": "false"
+ }
+ ]
+ }
+ },
+ {
+ "widget-type": "hidden",
+ "label": "Treat precision less number as Decimal(old behavior)",
+ "name": "treatPrecisionlessNumAsDeci",
+ "widget-attributes": {
+ "layout": "inline",
+ "default": "false",
+ "options": [
+ {
+ "id": "true",
+ "label": "true"
+ },
+ {
+ "id": "false",
+ "label": "false"
+ }
+ ]
+ }
+ },
{
"name": "connectionType",
"label": "Connection Type",
@@ -326,6 +364,14 @@
{
"type": "property",
"name": "transactionIsolationLevel"
+ },
+ {
+ "type": "property",
+ "name": "getTreatAsOldTimestampConn"
+ },
+ {
+ "type": "property",
+ "name": "treatPrecisionlessNumAsDeci"
}
]
},
diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json
index 628027caf..013f3b240 100644
--- a/oracle-plugin/widgets/Oracle-connector.json
+++ b/oracle-plugin/widgets/Oracle-connector.json
@@ -129,6 +129,44 @@
}
]
}
+ },
+ {
+ "widget-type": "hidden",
+ "label": "Treat as old timestamp",
+ "name": "treatAsOldTimestamp",
+ "widget-attributes": {
+ "layout": "inline",
+ "default": "false",
+ "options": [
+ {
+ "id": "true",
+ "label": "true"
+ },
+ {
+ "id": "false",
+ "label": "false"
+ }
+ ]
+ }
+ },
+ {
+ "widget-type": "hidden",
+ "label": "Treat precision less number as Decimal(old behavior)",
+ "name": "treatPrecisionlessNumAsDeci",
+ "widget-attributes": {
+ "layout": "inline",
+ "default": "false",
+ "options": [
+ {
+ "id": "true",
+ "label": "true"
+ },
+ {
+ "id": "false",
+ "label": "false"
+ }
+ ]
+ }
}
]
},
diff --git a/pom.xml b/pom.xml
index f09b52aad..45afac8ee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,7 +20,7 @@
io.cdap.plugin
database-plugins-parent
- 1.11.7
+ 1.11.8-SNAPSHOT
pom
Database Plugins
Collection of database plugins
diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml
index 90debbed9..bd66a7eb4 100644
--- a/postgresql-plugin/pom.xml
+++ b/postgresql-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
PostgreSQL plugin
diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml
index a740d7c15..2459b73c7 100644
--- a/saphana-plugin/pom.xml
+++ b/saphana-plugin/pom.xml
@@ -20,7 +20,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
SAP HANA plugin
diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml
index 7c9d7d3c3..797365865 100644
--- a/teradata-plugin/pom.xml
+++ b/teradata-plugin/pom.xml
@@ -21,7 +21,7 @@
database-plugins-parent
io.cdap.plugin
- 1.11.7
+ 1.11.8-SNAPSHOT
teradata-plugin