CDAP-15658 fixed tests for generic databases + added tests for schemas and fields validation

Author: mlozbin-cybervisiontech

README.md

@@ -5,21 +5,13 @@ machine or in docker container. Tests create tables and sample data, so user con
 corresponding permissions. To run integration tests execute following command in shell:
 ```
 mvn clean test \ 
--Dmysql.host=localhost -Dmysql.port=3306 -Dmysql.database=mydb -Dmysql.username=root -Dmysql.password=root \
--Dpostgresql.host=localhost -Dpostgresql.port=54032 -Dpostgresql.database=test -Dpostgresql.username=postgres \ 
--Dpostgresql.password=cdap \
--Doracle.host=localhost -Doracle.port=1521 -Doracle.username=ora -Doracle.password=cdap -Doracle.database=EE \
--Doracle.connectionType=SID \
--Ddb2.host=localhost -Ddb2.port=50000 -Ddb2.database=SAMPLE -Ddb2.username=DB2INST -Db2.password=DB2INST1-pwd \
--Dnetezza.host=localhost -Dnetezza.port=5480 -Dnetezza.database=test -Dnetezza.username=admin \
--Dnetezza.password=password \
 -DauroraMysql.clusterEndpoint=cdap-cluster.xyz.eu-central-1.rds.amazonaws.com -DauroraMysql.port=3306 \
 -DauroraMysql.database=cdapdb -DauroraMysql.username=cdap -DauroraMysql.password=cdap \
 -DauroraPostgresql.clusterEndpoint=pginstance.cxywmbgwp60k.eu-central-1.rds.amazonaws.com -DauroraPostgresql.port=5432 \
--DauroraPostgresql.database=cdappg -DauroraPostgresql.username=cdap -DauroraPostgresql.password=cdap
+-DauroraPostgresql.database=cdappg -DauroraPostgresql.username=cdap -DauroraPostgresql.password=cdap \
 -DjdbcDriversJars="/jdbc/drivers/jars/some.jar, "
 ```
-
+Notice that you must change properties for Aurora MySQL and Aurora Postgresql to real before running tests.
 ## Setup Local Environment
 MySQL, Postgresql, MSSQL, DB2 are using prebuild images.
 
@@ -36,43 +28,55 @@ cd docker-compose/db-plugins-env/
 docker-compose up -d
 ```
 * [Install and start Netezza emulator](http://dwgeek.com/install-vmware-player-netezza-emulator.html/)
+* Create database `mydb` in Netezza emulator
 
 ### Properties
 #### MySQL
 * **mysql.host** - Server host. Default: localhost.
 * **mysql.port** - Server port. Default: 3306.
 * **mysql.database** - Server namespace for test databases. Default: mydb.
-* **mysql.username** - Server username Default: root.
+* **mysql.username** - Server username. Default: root.
 * **mysql.password** - Server password. Default: 123Qwe123.
 #### Postgresql
 * **postgresql.host** - Server host. Default: localhost.
 * **postgresql.port** - Server port. Default: 5432.
 * **postgresql.database** - Server namespace for test databases. Default: mydb.
-* **postgresql.username** - Server username Default: postgres.
+* **postgresql.username** - Server username. Default: postgres.
 * **postgresql.password** - Server password. Default: 123Qwe123.
 #### MSSQL
 * **mssql.host** - Server host. Default: localhost.
 * **mssql.port** - Server port. Default: 1433.
 * **mssql.database** - Server namespace for test databases. Default: tempdb.
-* **mssql.username** - Server username Default: sa.
+* **mssql.username** - Server username. Default: sa.
 * **mssql.password** - Server password. Default: 123Qwe123.
 #### DB2
 * **db2.host** - Server host. Default: localhost.
 * **db2.port** - Server port. Default: 50000. 
 * **db2.database** - Server namespace for test databases. Default: mydb.
-* **db2.username** - Server username Default: db2inst1.
+* **db2.username** - Server username. Default: db2inst1.
 * **db2.password** - Server password. Default: 123Qwe123.
 #### Oracle
 * **oracle.host** - Server host. Default: localhost.
 * **oracle.port** - Server port. Default: 1521.
-* **oracle.username** - Server username Default: SYSTEM.
+* **oracle.username** - Server username. Default: SYSTEM.
 * **oracle.password** - Server password. Default: 123Qwe123.
 * **oracle.database** - Server sid/database. Default: cdap.
 * **oracle.connectionType** - Server connection type (service/sid) Default: sid.
 #### Netezza
 * **netezza.host** - Server host. Default: localhost.
 * **netezza.port** - Server port. Default: 5480.
 * **netezza.database** - Server namespace for test databases. Default: mydb.
-* **netezza.username** - Server username Default: admin.
+* **netezza.username** - Server username. Default: admin.
 * **netezza.password** - Server password. Default: password.
-  
+#### Aurora MySQL
+* **auroraMysql.clusterEndpoint** - Cluster endpoint.
+* **auroraMysql.port** - Server port.
+* **auroraMysql.database** - Server namespace for test databases.
+* **auroraMysql.username** - Server username.
+* **auroraMysql.password** - Server password.
+#### Aurora Postgresql
+* **auroraPostgresql.clusterEndpoint** - Cluster endpoint.
+* **auroraPostgresql.port** - Server port.
+* **auroraPostgresql.database** - Server namespace for test databases. Default: mydb.
+* **auroraPostgresql.username** - Server username.
+* **auroraPostgresql.password** - Server password.  

aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestBase.java

@@ -55,7 +55,7 @@
 import java.util.TimeZone;
 import javax.sql.rowset.serial.SerialBlob;
 
-public class AuroraMysqlPluginTestBase extends DatabasePluginTestBase {
+public abstract class AuroraMysqlPluginTestBase extends DatabasePluginTestBase {
   protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0");
   protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0");
   protected static final long CURRENT_TS = System.currentTimeMillis();
@@ -73,11 +73,11 @@ public class AuroraMysqlPluginTestBase extends DatabasePluginTestBase {
   public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false);
 
   protected static final Map<String, String> BASE_PROPS = ImmutableMap.<String, String>builder()
-    .put(ConnectionConfig.HOST, System.getProperty("auroraMysql.clusterEndpoint"))
-    .put(ConnectionConfig.PORT, System.getProperty("auroraMysql.port"))
-    .put(ConnectionConfig.DATABASE, System.getProperty("auroraMysql.database"))
-    .put(ConnectionConfig.USER, System.getProperty("auroraMysql.username"))
-    .put(ConnectionConfig.PASSWORD, System.getProperty("auroraMysql.password"))
+    .put(ConnectionConfig.HOST, getPropertyOrFail("auroraMysql.clusterEndpoint"))
+    .put(ConnectionConfig.PORT, getPropertyOrFail("auroraMysql.port"))
+    .put(ConnectionConfig.DATABASE, getPropertyOrFail("auroraMysql.database"))
+    .put(ConnectionConfig.USER, getPropertyOrFail("auroraMysql.username"))
+    .put(ConnectionConfig.PASSWORD, getPropertyOrFail("auroraMysql.password"))
     .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME)
     .build();
 
@@ -185,7 +185,7 @@ protected static void prepareTestData(Connection conn) throws SQLException {
     }
   }
 
-  private static void populateData(PreparedStatement ...stmts) throws SQLException {
+  private static void populateData(PreparedStatement... stmts) throws SQLException {
     // insert the same data into both tables: my_table and your_table
     for (PreparedStatement pStmt : stmts) {
       for (int i = 1; i <= 5; i++) {
@@ -228,6 +228,16 @@ private static void populateData(PreparedStatement ...stmts) throws SQLException
     }
   }
 
+  private static String getPropertyOrFail(String propertyName) {
+    String value = System.getProperty(propertyName);
+
+    if (value == null) {
+      throw new IllegalStateException("There is no value for property " + propertyName);
+    }
+
+    return value;
+  }
+
   public static Connection createConnection() {
     try {
       Class.forName(DRIVER_CLASS);

aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestBase.java

@@ -50,7 +50,7 @@
 import java.util.Map;
 import java.util.TimeZone;
 
-public class AuroraPostgresPluginTestBase extends DatabasePluginTestBase {
+public abstract class AuroraPostgresPluginTestBase extends DatabasePluginTestBase {
   protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0");
   protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0");
   protected static final long CURRENT_TS = System.currentTimeMillis();
@@ -68,11 +68,11 @@ public class AuroraPostgresPluginTestBase extends DatabasePluginTestBase {
   public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false);
 
   protected static final Map<String, String> BASE_PROPS = ImmutableMap.<String, String>builder()
-    .put(ConnectionConfig.HOST, System.getProperty("auroraPostgresql.clusterEndpoint"))
-    .put(ConnectionConfig.PORT, System.getProperty("auroraPostgresql.port"))
-    .put(ConnectionConfig.DATABASE, System.getProperty("auroraPostgresql.database"))
-    .put(ConnectionConfig.USER, System.getProperty("auroraPostgresql.username"))
-    .put(ConnectionConfig.PASSWORD, System.getProperty("auroraPostgresql.password"))
+    .put(ConnectionConfig.HOST, getPropertyOrFail("auroraPostgresql.clusterEndpoint"))
+    .put(ConnectionConfig.PORT, getPropertyOrFail("auroraPostgresql.port"))
+    .put(ConnectionConfig.DATABASE, getPropertyOrFail("auroraPostgresql.database"))
+    .put(ConnectionConfig.USER, getPropertyOrFail("auroraPostgresql.username"))
+    .put(ConnectionConfig.PASSWORD, getPropertyOrFail("auroraPostgresql.password"))
     .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME)
     .build();
 
@@ -163,7 +163,7 @@ protected static void prepareTestData(Connection conn) throws SQLException {
     }
   }
 
-  private static void populateData(PreparedStatement ...stmts) throws SQLException {
+  private static void populateData(PreparedStatement... stmts) throws SQLException {
     // insert the same data into both tables: my_table and your_table
     for (PreparedStatement pStmt : stmts) {
       for (int i = 1; i <= 5; i++) {
@@ -189,6 +189,16 @@ private static void populateData(PreparedStatement ...stmts) throws SQLException
     }
   }
 
+  private static String getPropertyOrFail(String propertyName) {
+    String value = System.getProperty(propertyName);
+
+    if (value == null) {
+      throw new IllegalStateException("There is no value for property " + propertyName);
+    }
+
+    return value;
+  }
+
   public static Connection createConnection() {
     try {
       Class.forName(Driver.class.getCanonicalName());

database-commons/pom.xml

@@ -59,6 +59,12 @@
       <artifactId>cdap-api</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>com.mockrunner</groupId>
+      <artifactId>mockrunner-jdbc</artifactId>
+      <version>${mockrunner.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

database-commons/src/main/java/io/cdap/plugin/db/ColumnType.java

@@ -16,6 +16,8 @@
 
 package io.cdap.plugin.db;
 
+import java.util.Objects;
+
 /**
  * Stores SQL column name and type.
  */
@@ -42,4 +44,32 @@ public String getTypeName() {
   public int getType() {
     return type;
   }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    ColumnType that = (ColumnType) o;
+    return type == that.type &&
+      Objects.equals(name, that.name) &&
+      Objects.equals(typeName, that.typeName);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(name, typeName, type);
+  }
+
+  @Override
+  public String toString() {
+    return "ColumnType{" +
+      "name='" + name + '\'' +
+      ", typeName='" + typeName + '\'' +
+      ", type=" + type +
+      '}';
+  }
 }