diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/Datatypes.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/Datatypes.feature new file mode 100644 index 000000000..94ec241a5 --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/Datatypes.feature @@ -0,0 +1,56 @@ +# +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +@Mysql +Feature: Mysql - Verify Mysql sink data transfer for different datatypes + + @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE + Scenario: To verify data is getting transferred from BigQuery to Mysql successfully + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqSourceTable" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "MySQL" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target MySQL table is equal to the values from source BigQuery table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTime.feature new file mode 100644 index 000000000..008a8e4dc --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTime.feature @@ -0,0 +1,77 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License... + +@Mysql +Feature: MySQL Sink - Design time scenarios + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify user can validate successfully when sink plugin is configured for table with basic details + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + + @MYSQL_SOURCE_TEST @CONNECTION @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify the MySQL Sink plugin using connection manager functionality + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "MySQL" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + And Select Mysql Connection + And Enter input plugin property: "name" with value: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify user should be able to validate sink plugin successfully by setting the advanced section fields + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click plugin property: "useCompression" + Then Click plugin property: "autoReconnect" + Then Validate "MySQL" plugin properties diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature new file mode 100644 index 000000000..26e80bd39 --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature @@ -0,0 +1,169 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License + +@Mysql +Feature: MySQL Sink - Design time validation scenarios + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify Database field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "invalid.database" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "invalid.sink.database.message" on the header + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify Table Name Field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "invalid.tablename" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin Property: "tableName" is displaying an in-line error message: "invalidtableName.error.message" + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify Reference Name field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "invalid.referenceName" + Then Click on the Validate button + Then Verify that the Plugin Property: "referenceName" is displaying an in-line error message: "invalidreferenceName.error.message" + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify the Username field validation error message with blank value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin Property: "user" is displaying an in-line error message: "blank.username.message" + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify the host validation error message with blank value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "invalid.host.message" on the header diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeWithMacros.feature new file mode 100644 index 000000000..ddc7cfb14 --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeWithMacros.feature @@ -0,0 +1,76 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Mysql +Feature: MySQL Sink - Design time scenarios (macro) + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify user should be able to validate sink plugin with macros for Connection section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Click on the Validate button + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify user should be able to validate sink plugin with macros for Basic section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Validate button + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + And Click on the Macro button of Property: "tableName" and set the value to: "targetTable" + And Click on the Macro button of Property: "database" and set the value to: "databaseName" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Validate button diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature new file mode 100644 index 000000000..2bcca67c2 --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature @@ -0,0 +1,149 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Mysql +Feature: MySQL Sink - Run time scenarios + + @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @Mysql_Required @Plugin-20670 + Scenario: To verify data is getting transferred from BigQuery to Mysql successfully + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchema" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "MySQL" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target MySQL table is equal to the values from source BigQuery table + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully when advance section details are set + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click plugin property: "useCompression" + Then Click plugin property: "useAnsiQuotes" + Then Click plugin property: "autoReconnect" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature new file mode 100644 index 000000000..233226ea4 --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature @@ -0,0 +1,138 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Mysql +Feature: MySQL Sink - Run time scenarios (macro) + + @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @Plugin-20670 + Scenario: Verify user should be able to preview a pipeline when plugin is configured or fetching table details using macros. + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "project.id" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqSourceTable" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "MySQL" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Click on the Validate button + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + And Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value from environment variable "host" for key "Host" + And Enter runtime argument value from environment variable "port" for key "Port" + And Enter runtime argument value from environment variable "username" for key "Username" + And Enter runtime argument value from environment variable "password" for key "Password" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + And Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value from environment variable "host" for key "Host" + And Enter runtime argument value from environment variable "port" for key "Port" + And Enter runtime argument value from environment variable "username" for key "Username" + And Enter runtime argument value from environment variable "password" for key "Password" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target Big Query table is equal to the values from source table + + @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @Mysql_Required + Scenario: Verify that the pipeline fails when user provides invalid Table Name of plugin with Macros + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "project.id" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqSourceTable" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "MySQL" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + And Click on the Macro button of Property: "tableName" and set the value to: "tableName" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Deploy the pipeline + Then Run the Pipeline in Runtime + And Enter runtime argument value "invalid.tableName" for key "tableName" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + And Verify the pipeline status is "Failed" + + @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @Mysql_Required + Scenario: Verify that the pipeline fails when user provides invalid Credentials for connection with Macros + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "project.id" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqSourceTable" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "MySQL" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "username" + And Click on the Macro button of Property: "password" and set the value to: "password" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Close the Plugin Properties page + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Enter runtime argument value "invalid.host" for key "Host" + And Enter runtime argument value "invalid.port" for key "Port" + And Enter runtime argument value "invalid.username" for key "username" + And Enter runtime argument value "invalid.password" for key "password" + And Run the Pipeline in Runtime with runtime arguments + And Wait till pipeline is in running state + And Verify the pipeline status is "Failed" diff --git a/mysql-plugin/src/e2e-test/features/mysql/Datatypes.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/Datatypes.feature similarity index 59% rename from mysql-plugin/src/e2e-test/features/mysql/Datatypes.feature rename to mysql-plugin/src/e2e-test/features/mysqlsource/Datatypes.feature index c657fdd59..03421d527 100644 --- a/mysql-plugin/src/e2e-test/features/mysql/Datatypes.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/Datatypes.feature @@ -1,5 +1,5 @@ # -# Copyright © 2022 Cask Data, Inc. +# Copyright © 2023 Cask Data, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of @@ -15,7 +15,8 @@ # @Mysql -Feature: Mysql - Verify Mysql source data transfer for different datatypes +Feature: Mysql - Verify Mysql source data transfer for different datatype + @MYSQL_SOURCE_DATATYPES_TEST @MYSQL_SINK_TEST @Mysql_Required Scenario: To verify data is getting transferred from Mysql to Mysql successfully Given Open Datafusion Project to configure pipeline @@ -59,3 +60,40 @@ Feature: Mysql - Verify Mysql source data transfer for different datatypes Then Open and capture logs Then Verify the pipeline status is "Succeeded" Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_DATATYPES_TEST @BQ_SINK @Mysql_Required @Plugin-20670 + Scenario: To verify data is getting transferred from Mysql to BigQuery successfully + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "BigQuery" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "BigQuery" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqTargetTable" + Then Click plugin property: "truncateTable" + Then Click plugin property: "updateTableSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target Big Query table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTime.feature new file mode 100644 index 000000000..2d1c55d3c --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTime.feature @@ -0,0 +1,102 @@ +# Copyright © 2023 Cask Data, Inc. +## +## Licensed under the Apache License, Version 2.0 (the "License"); you may not +## use this file except in compliance with the License. You may obtain a copy of +## the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +## License for the specific language governing permissions and limitations under +# the License.. + +@Mysql +Feature: MySQL Source - Design time scenarios + + @MYSQL_SOURCE_TEST + Scenario: Verify user is able to validate the plugin when configured for basic Section. + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + + @MYSQL_SOURCE_TEST @CONNECTION + Scenario: Verify user is able validate the plugin with connection management functionality + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + And Select Mysql Connection + And Enter input plugin property: "name" with value: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + + @MYSQL_SOURCE_TEST + Scenario: Verify user should be able to validate successfully by setting the advanced section fields + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click plugin property: "useCompression" + Then Click plugin property: "useAnsiQuotes" + Then Click plugin property: "autoReconnect" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + + @MYSQL_SOURCE_TEST + Scenario: To verify Mysql source plugin validation with connection arguments + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsList" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature new file mode 100644 index 000000000..ee317eac6 --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature @@ -0,0 +1,143 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License + +@Mysql +Feature: MySQL Source - Design time validation scenarios + + Scenario: Verify UserName field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "invalid.username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "invalid.username.message" on the header + + Scenario: Verify Password field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "invalid.password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "invalid.password.message" on the header + + Scenario: Verify Database field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "invalid.database" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "invalid.databasename.message" on the header + + Scenario: Verify ImportQuery Field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "invalid.query" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "invalid.query.message" on the header + + Scenario: Verify the Number of Splits to generate field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "numSplits" with value: "numberofsplits" + Then Click on the Validate button + Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "numberofsplits.error.message" + + Scenario: Verify the Bounding Query validation error when Split-By and Number of Splits values are not provided + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "numSplits" with value: "numberofsplitsgenerate" + Then Click on the Validate button + Then Verify that the Plugin Property: "boundingQuery" is displaying an in-line error message: "boundingQuery.error.message" + + Scenario: Verify the Split-By field validation error message with invalid test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "numSplits" with value: "numbersplitsgenerate" + Then Click on the Validate button + Then Verify that the Plugin Property: "splitBy" is displaying an in-line error message: "splitfield.error.message" + + Scenario: Verify required fields missing validation messages + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | jdbcPluginName | + | referenceName | + | database | + | importQuery | diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeWithMacros.feature new file mode 100644 index 000000000..d64738e5e --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeWithMacros.feature @@ -0,0 +1,48 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Mysql +Feature: MySQL Source - Design time scenarios (macro) + + Scenario: Verify user should be able to validate plugin with macros for Connection section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + + Scenario: Verify user should be able to validate plugin with macros for Basic section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + And Click on the Macro button of Property: "database" and set the value to: "databaseName" + And Click on the Macro button of Property: "fetchSize" and set the value to: "fetchsize" + And Click on the Macro button of Property: "splitBy" and set the value to: "splitbyfield" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties diff --git a/mysql-plugin/src/e2e-test/features/mysql/MySql.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/MySql.feature similarity index 99% rename from mysql-plugin/src/e2e-test/features/mysql/MySql.feature rename to mysql-plugin/src/e2e-test/features/mysqlsource/MySql.feature index e2515f697..c2244174b 100644 --- a/mysql-plugin/src/e2e-test/features/mysql/MySql.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/MySql.feature @@ -1,5 +1,5 @@ # -# Copyright © 2022 Cask Data, Inc. +# Copyright © 2023 Cask Data, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of @@ -16,6 +16,7 @@ @Mysql Feature: Mysql - Verify Mysql source data transfer + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required Scenario: To verify data is getting transferred from Mysql to Mysql successfully Given Open Datafusion Project to configure pipeline diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature new file mode 100644 index 000000000..8d5e5a40e --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature @@ -0,0 +1,190 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Mysql +Feature: MySQL Source - Run time scenarios + + @MYSQL_SOURCE_TEST @BQ_SINK @BQ_SINK_CLEANUP @Plugin-20670 + Scenario: To verify data is getting transferred from Mysql to BigQuery successfully + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + And Select Sink plugin: "BigQueryTable" from the plugins list + Then Connect plugins: "MySQL" and "BigQuery" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqTargetTable" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + Then Save the pipeline + And Preview and run the pipeline + And Wait till pipeline preview is in running state + And Open and capture pipeline preview logs + And Verify the preview run status of pipeline in the logs is "succeeded" + And Close the pipeline logs + And Close the preview + And Deploy the pipeline + And Run the Pipeline in Runtime + And Wait till pipeline is in running state + And Open and capture logs + And Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate OUT record count is equal to records transferred to target BigQuery table + Then Validate the values of records transferred to target Big Query table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required @test + Scenario: To verify data is getting transferred from Mysql to Mysql successfully when advance section details are set + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click plugin property: "useCompression" + Then Click plugin property: "useAnsiQuotes" + Then Click plugin property: "autoReconnect" + Then Click on the Get Schema button + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify the pipeline fails when plugin is configured with invalid bounding query + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Enter textarea plugin property: "boundingQuery" with value: "invalid.boundQuery" + Then Enter textarea plugin property: "importQuery" with value: "ImportQuery" + Then Replace input plugin property: "splitBy" with value: "splitby" + Then Replace input plugin property: "numSplits" with value: "numbersplitsgenerate" + Then Click on the Get Schema button + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Verify the preview run status of pipeline in the logs is "failed" + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully when connection arguments are set + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsList" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table \ No newline at end of file diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature new file mode 100644 index 000000000..88d5b5b41 --- /dev/null +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature @@ -0,0 +1,244 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Mysql +Feature: MySQL Source - Run time scenarios (macro) + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully using macros for Connection section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + And Preview and run the pipeline + And Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value from environment variable "host" for key "Host" + And Enter runtime argument value from environment variable "port" for key "Port" + And Enter runtime argument value from environment variable "username" for key "Username" + And Enter runtime argument value from environment variable "password" for key "Password" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + And Close the preview + And Deploy the pipeline + And Run the Pipeline in Runtime + And Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value from environment variable "host" for key "Host" + And Enter runtime argument value from environment variable "port" for key "Port" + And Enter runtime argument value from environment variable "username" for key "Username" + And Enter runtime argument value from environment variable "password" for key "Password" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully using macros for Basic section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + And Click on the Macro button of Property: "database" and set the value to: "databaseName" + And Click on the Macro button of Property: "fetchSize" and set the value to: "fetchSize" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + And Enter runtime argument value "databaseName" for key "databaseName" + And Enter runtime argument value "fetchsize" for key "fetchSize" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + And Enter runtime argument value "databaseName" for key "databaseName" + And Enter runtime argument value "fetchsize" for key "fetchSize" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @BQ_SINK @BQ_SINK_CLEANUP @Plugin-20670 + Scenario: To verify data is getting transferred from Mysql to BigQuery successfully using macros for Connection section + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + And Select Sink plugin: "BigQueryTable" from the plugins list + Then Connect plugins: "MySQL" and "BigQuery" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + And Click on the Macro button of Property: "jdbcPluginName" and set the value to: "DriverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqTargetTable" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + And Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value from environment variable "host" for key "Host" + And Enter runtime argument value from environment variable "port" for key "Port" + And Enter runtime argument value from environment variable "username" for key "Username" + And Enter runtime argument value from environment variable "password" for key "Password" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + And Enter runtime argument value "driverName" for key "DriverName" + And Enter runtime argument value from environment variable "host" for key "Host" + And Enter runtime argument value from environment variable "port" for key "Port" + And Enter runtime argument value from environment variable "username" for key "Username" + And Enter runtime argument value from environment variable "password" for key "Password" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target Big Query table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify that pipeline fails when user provides invalid Table name in importQuery of plugin with Macros + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + And Click on the Macro button of Property: "importQuery" and set the value in textarea: "importQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Enter runtime argument value "invalid.query" for key "importQuery" + And Run the Pipeline in Runtime with runtime arguments + And Wait till pipeline is in running state + And Verify the pipeline status is "Failed" + + @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST @Mysql_Required + Scenario: Verify that pipeline fails when user provides invalid Credentials for connection with Macros + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + And Click on the Macro button of Property: "host" and set the value to: "Host" + And Click on the Macro button of Property: "port" and set the value to: "Port" + And Click on the Macro button of Property: "user" and set the value to: "Username" + And Click on the Macro button of Property: "password" and set the value to: "Password" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Enter runtime argument value "invalid.host" for key "Host" + And Enter runtime argument value "invalid.port" for key "Port" + And Enter runtime argument value "invalid.username" for key "Username" + And Enter runtime argument value "invalid.password" for key "Password" + And Run the Pipeline in Runtime with runtime arguments + And Wait till pipeline is in running state + And Verify the pipeline status is "Failed" diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java index c4064e885..d80e4dfc4 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java @@ -33,15 +33,15 @@ import java.util.TimeZone; /** - * MySQL client. + * MySQL client. */ public class MysqlClient { private static final String database = PluginPropertyUtils.pluginProp("databaseName"); - private static Connection getMysqlConnection() throws SQLException, ClassNotFoundException { + public static Connection getMysqlConnection() throws SQLException, ClassNotFoundException { Class.forName("com.mysql.cj.jdbc.Driver"); return DriverManager.getConnection("jdbc:mysql://" + System.getenv("MYSQL_HOST") + ":" + - System.getenv("MYSQL_PORT") + "/" + database, + System.getenv("MYSQL_PORT") + "/" + database + "?tinyInt1isBit=false", System.getenv("MYSQL_USERNAME"), System.getenv("MYSQL_PASSWORD")); } @@ -60,6 +60,7 @@ public static int countRecord(String table) throws SQLException, ClassNotFoundEx /** * Extracts entire data from source and target tables. + * * @param sourceTable table at the source side * @param targetTable table at the sink side * @return true if the values in source and target side are equal @@ -82,6 +83,7 @@ public static boolean validateRecordValues(String sourceTable, String targetTabl /** * Compares the result Set data in source table and sink table.. + * * @param rsSource result set of the source table data * @param rsTarget result set of the target table data * @return true if rsSource matches rsTarget @@ -106,20 +108,20 @@ public static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarge Timestamp targetTS = rsTarget.getTimestamp(currentColumnCount, gc); if (sourceTS == null) { Assert.assertNull(String.format("Not null TIMESTAMP value found in target for column : %s", columnName), - targetTS); + targetTS); } else { Assert.assertEquals(String.format("Different TIMESTAMP values found for column : %s", columnName), - sourceTS, targetTS); + sourceTS, targetTS); } } else { String sourceString = rsSource.getString(currentColumnCount); String targetString = rsTarget.getString(currentColumnCount); if (Strings.isNullOrEmpty(sourceString)) { Assert.assertTrue(String.format("Not null/empty value found in target for column : %s", columnName), - Strings.isNullOrEmpty(targetString)); + Strings.isNullOrEmpty(targetString)); } else { Assert.assertEquals(String.format("Different values found for column : %s", columnName), - sourceString, targetString); + sourceString, targetString); } } currentColumnCount++; @@ -165,6 +167,18 @@ public static void createTargetTable(String targetTable) throws SQLException, Cl } } + public static void createTargetTable1(String targetTable) throws SQLException, ClassNotFoundException { + try (Connection connect = getMysqlConnection(); + Statement statement = connect.createStatement()) { + String createTargetTableQuery = "CREATE TABLE IF NOT EXISTS " + targetTable + + "(id bigint(20),lastName varchar(255))"; + statement.executeUpdate(createTargetTableQuery); + // Truncate table to clean the data of last failure run. + String truncateTargetTableQuery = "TRUNCATE TABLE " + targetTable; + statement.executeUpdate(truncateTargetTableQuery); + } + } + public static void createSourceDatatypesTable(String sourceTable) throws SQLException, ClassNotFoundException { try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) { @@ -201,4 +215,13 @@ public static void dropTables(String[] tables) throws SQLException, ClassNotFoun } } } + + public static void dropTable(String table) throws SQLException, ClassNotFoundException { + try (Connection connect = getMysqlConnection(); + Statement statement = connect.createStatement()) { + String dropTableQuery = "Drop Table " + table; + statement.executeUpdate(dropTableQuery); + } + } } + diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 9175e9c8d..4c7ba05ae 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021 Cask Data, Inc. + * Copyright © 2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -16,18 +16,30 @@ package io.cdap.plugin.common.stepsdesign; +import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.MysqlClient; import io.cucumber.java.After; import io.cucumber.java.Before; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import stepsdesign.BeforeActions; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; import java.sql.SQLException; +import java.util.NoSuchElementException; +import java.util.UUID; /** * MYSQL test hooks. */ public class TestSetupHooks { + public static String bqTargetTable = StringUtils.EMPTY; private static void setTableName() { String randomString = RandomStringUtils.randomAlphabetic(10); @@ -69,4 +81,95 @@ public static void dropTables() throws SQLException, ClassNotFoundException { PluginPropertyUtils.pluginProp("targetTable")}); } + @Before(order = 1, value = "@BQ_SINK") + public static void setTempTargetBQTable() { + bqTargetTable = "TestSN_table" + RandomStringUtils.randomAlphanumeric(10); + PluginPropertyUtils.addPluginProp("bqTargetTable", bqTargetTable); + BeforeActions.scenario.write("BigQuery Target table name: " + bqTargetTable); + } + + @After(order = 1, value = "@BQ_SINK_CLEANUP") + public static void deleteTempTargetBQTable() throws IOException, InterruptedException { + try { + BigQueryClient.dropBqQuery(bqTargetTable); + BeforeActions.scenario.write("BigQuery Target table: " + bqTargetTable + " is deleted successfully"); + bqTargetTable = StringUtils.EMPTY; + } catch (BigQueryException e) { + if (e.getCode() == 404) { + BeforeActions.scenario.write("BigQuery Target Table: " + bqTargetTable + " does not exist"); + } else { + Assert.fail(e.getMessage()); + } + } + } + + @Before(order = 1, value = "@MYSQL_TARGET_TABLE") + public static void createMysqlTargetTable() throws SQLException, ClassNotFoundException { + MysqlClient.createTargetTable1(PluginPropertyUtils.pluginProp("targetTable")); + } + + @After(order = 2, value = "@MYSQL_TARGET_TABLE") + public static void dropTargetTable() throws SQLException, ClassNotFoundException { + MysqlClient.dropTable(PluginPropertyUtils.pluginProp("targetTable")); + } + + + + /** + * Create BigQuery table. + */ + + @Before(order = 1, value = "@BQ_SOURCE_TEST") + public static void createTempSourceBQTable() throws IOException, InterruptedException { + createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFile"), + PluginPropertyUtils.pluginProp("InsertBQDataQueryFile")); + } + @After(order = 1, value = "@BQ_SOURCE_TEST") + private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile, String bqInsertDataQueryFile) + throws IOException, InterruptedException { + String bqSourceTable = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_"); + + String createTableQuery = StringUtils.EMPTY; + try { + createTableQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource + ("/" + bqCreateTableQueryFile).toURI())) + , StandardCharsets.UTF_8); + createTableQuery = createTableQuery.replace("DATASET", PluginPropertyUtils.pluginProp("dataset")) + .replace("TABLE_NAME", bqSourceTable); + } catch (Exception e) { + BeforeActions.scenario.write("Exception in reading " + bqCreateTableQueryFile + " - " + e.getMessage()); + Assert.fail("Exception in BigQuery testdata prerequisite setup " + + "- error in reading create table query file " + e.getMessage()); + } + + String insertDataQuery = StringUtils.EMPTY; + try { + insertDataQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource + ("/" + bqInsertDataQueryFile).toURI())) + , StandardCharsets.UTF_8); + insertDataQuery = insertDataQuery.replace("DATASET", PluginPropertyUtils.pluginProp("dataset")) + .replace("TABLE_NAME", bqSourceTable); + } catch (Exception e) { + BeforeActions.scenario.write("Exception in reading " + bqInsertDataQueryFile + " - " + e.getMessage()); + Assert.fail("Exception in BigQuery testdata prerequisite setup " + + "- error in reading insert data query file " + e.getMessage()); + } + BigQueryClient.getSoleQueryResult(createTableQuery); + + try { + BigQueryClient.getSoleQueryResult(insertDataQuery); + } catch (NoSuchElementException e) { + // Insert query does not return any record. + // Iterator on TableResult values in getSoleQueryResult method throws NoSuchElementException. + } + PluginPropertyUtils.addPluginProp("bqSourceTable", bqSourceTable); + BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " created successfully"); + } + + @Before(order = 1, value = "@CONNECTION") + public static void setNewConnectionName() { + String connectionName = "Mysql" + RandomStringUtils.randomAlphanumeric(10); + PluginPropertyUtils.addPluginProp("connection.name", connectionName); + BeforeActions.scenario.write("New Connection name: " + connectionName); + } } diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java index 3db70059a..63f8efabc 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Cask Data, Inc. + * Copyright © 2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/BQValidation.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/BQValidation.java new file mode 100644 index 000000000..a4d6a9bf8 --- /dev/null +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/BQValidation.java @@ -0,0 +1,229 @@ +package io.cdap.plugin.mysql; + +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import com.google.cloud.bigquery.TableResult; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import io.cdap.e2e.utils.BigQueryClient; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.plugin.MysqlClient; +import org.junit.Assert; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.sql.Types; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +/** + * BQValidation. + */ +public class BQValidation { + + /** + * Extracts entire data from source and target tables. + * + * @param sourceTable table at the source side + * @param targetTable table at the BigQuery side + * @return true if the values in source and target side are equal + */ + + public static boolean validateBQAndDBRecordValues(String sourceTable, String targetTable) + throws SQLException, ClassNotFoundException, ParseException, IOException, InterruptedException { + List jsonResponse = new ArrayList<>(); + List bigQueryRows = new ArrayList<>(); + getBigQueryTableData(targetTable, bigQueryRows); + for (Object rows : bigQueryRows) { + JsonObject json = new Gson().fromJson(String.valueOf(rows), JsonObject.class); + jsonResponse.add(json); + } + String getSourceQuery = "SELECT * FROM " + sourceTable; + System.out.println(getSourceQuery); + try (Connection connect = MysqlClient.getMysqlConnection()) { + connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); + Statement statement1 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE, + ResultSet.HOLD_CURSORS_OVER_COMMIT); + + ResultSet rsSource = statement1.executeQuery(getSourceQuery); + return compareResultSetAndJsonData(rsSource, jsonResponse); + } + } + + /** + * Retrieves the data from a specified BigQuery table and populates it into the provided list of objects. + * + * @param table The name of the BigQuery table to fetch data from. + * @param bigQueryRows The list to store the fetched BigQuery data. + */ + + private static void getBigQueryTableData(String table, List bigQueryRows) + throws IOException, InterruptedException { + + String projectId = PluginPropertyUtils.pluginProp("projectId"); + String dataset = PluginPropertyUtils.pluginProp("dataset"); + String selectQuery = "SELECT TO_JSON(t) FROM `" + projectId + "." + dataset + "." + table + "` AS t"; + TableResult result = BigQueryClient.getQueryResult(selectQuery); + result.iterateAll().forEach(value -> bigQueryRows.add(value.get(0).getValue())); + } + + /** + * Compares the data in the result set obtained from the Oracle database with the provided BigQuery JSON objects. + * + * @param rsSource The result set obtained from the Oracle database. + * @param bigQueryData The list of BigQuery JSON objects to compare with the result set data. + * @return True if the result set data matches the BigQuery data, false otherwise. + * @throws SQLException If an SQL error occurs during the result set operations. + * @throws ParseException If an error occurs while parsing the data. + */ + + public static boolean compareResultSetAndJsonData(ResultSet rsSource, List bigQueryData) + throws SQLException, + ParseException { + ResultSetMetaData mdSource = rsSource.getMetaData(); + boolean result = false; + int columnCountSource = mdSource.getColumnCount(); + + if (bigQueryData == null) { + Assert.fail("bigQueryData is null"); + return result; + } + // Get the column count of the first JsonObject in bigQueryData + int columnCountTarget = 0; + if (bigQueryData.size() > 0) { + columnCountTarget = bigQueryData.get(0).entrySet().size(); + } + // Compare the number of columns in the source and target + Assert.assertEquals("Number of columns in source and target are not equal", + columnCountSource, columnCountTarget); + + //Variable 'jsonObjectIdx' to track the index of the current JsonObject in the bigQueryData list, + int jsonObjectIdx = 0; + while (rsSource.next()) { + int currentColumnCount = 1; + while (currentColumnCount <= columnCountSource) { + String columnTypeName = mdSource.getColumnTypeName(currentColumnCount); + int columnType = mdSource.getColumnType(currentColumnCount); + String columnName = mdSource.getColumnName(currentColumnCount); + // Perform different comparisons based on column type + switch (columnType) { + case Types.BLOB: + case Types.VARBINARY: + case Types.LONGVARBINARY: + String sourceB64String = new String(Base64.getEncoder() + .encode(rsSource.getBytes(currentColumnCount))); + String targetB64String = bigQueryData.get(jsonObjectIdx).get(columnName).getAsString(); + Assert.assertEquals("Different values found for column : %s", + sourceB64String, targetB64String); + break; + + case Types.NUMERIC: + long sourceVal = rsSource.getLong(currentColumnCount); + long targetVal = Long.parseLong(bigQueryData.get(jsonObjectIdx).get(columnName).getAsString()); + Assert.assertEquals("Different values found for column : %s", String.valueOf(sourceVal), + String.valueOf(targetVal)); + break; + + case Types.TIMESTAMP: + Timestamp sourceTS = rsSource.getTimestamp(columnName); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date parsedDate = dateFormat. + parse(bigQueryData.get(jsonObjectIdx).get(columnName).getAsString()); + Timestamp targetTs = new java.sql.Timestamp(parsedDate.getTime()); + + Assert.assertEquals("Different values found for column: %s", + sourceTS, targetTs); + break; + + case Types.FLOAT: + Float sourceBytes = rsSource.getFloat(currentColumnCount); + Float targetBytes = bigQueryData.get(jsonObjectIdx).get(columnName).getAsFloat(); + Assert.assertEquals(String.format("Different values found for column : %s", columnName), 0, + Float.compare(sourceBytes, targetBytes)); + break; + + case Types.DOUBLE: + double sourceVAL = rsSource.getDouble(currentColumnCount); + double targetVAL = bigQueryData.get(jsonObjectIdx).get(columnName).getAsDouble(); + Assert.assertEquals(String.format("Different values found for column : %s", columnName), 0, + Double.compare(sourceVAL, targetVAL)); + break; + + case Types.BINARY: + byte sourceBit = rsSource.getByte(currentColumnCount); + boolean sourceBitValue = sourceBit == 1; + boolean targetBitValue = bigQueryData.get(jsonObjectIdx).get(columnName).getAsBoolean(); + Assert.assertEquals(String.format("Different binary values found for column: %s", columnName), + sourceBitValue, targetBitValue); + break; + + case Types.BIT: + byte source = rsSource.getByte(currentColumnCount); + boolean target = bigQueryData.get(jsonObjectIdx).get(columnName).getAsBoolean(); + boolean sourceAsBoolean = source != 0; + Assert.assertEquals(String.format("Different bit values found for column: %s", columnName), + sourceAsBoolean, target); + break; + + case Types.TINYINT: + default: + String sourceValue = rsSource.getString(currentColumnCount); + String targetValue = bigQueryData.get(jsonObjectIdx).get(columnName).getAsString(); + Assert.assertEquals + (String.format("Different %s values found for column : %s", columnTypeName, columnName), + String.valueOf(sourceValue), String.valueOf(targetValue)); + } + currentColumnCount++; + } + jsonObjectIdx++; + } + Assert.assertFalse("Number of rows in Source table is greater than the number of rows in Target table", + rsSource.next()); + return true; + } + + public static boolean validateBQToDBRecordValues(String sourceTable, String targetTable) + throws SQLException, ClassNotFoundException, ParseException, IOException, InterruptedException { + List jsonResponse = new ArrayList<>(); + List bigQueryRows = new ArrayList<>(); + getBigQueryTableData(sourceTable, bigQueryRows); + for (Object rows : bigQueryRows) { + JsonObject json = new Gson().fromJson(String.valueOf(rows), JsonObject.class); + jsonResponse.add(json); + } + String getTargetQuery = "SELECT * FROM " + targetTable; + try (Connection connect = MysqlClient.getMysqlConnection()) { + connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); + Statement statement1 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE, + ResultSet.HOLD_CURSORS_OVER_COMMIT); + + ResultSet rsTarget = statement1.executeQuery(getTargetQuery); + return compareResultSetAndJsonData(rsTarget, jsonResponse); + } + } +} diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/actions/MySQLPropertiesPageActions.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/actions/MySQLPropertiesPageActions.java new file mode 100644 index 000000000..f59cf94e4 --- /dev/null +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/actions/MySQLPropertiesPageActions.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql.actions; + +import io.cdap.e2e.utils.ElementHelper; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.e2e.utils.SeleniumHelper; +import io.cdap.plugin.mysql.locators.MySQLPropertiesPage; + +/** + * MySQL source - Properties page - Actions. + */ +public class MySQLPropertiesPageActions { + static { + SeleniumHelper.getPropertiesLocators(MySQLPropertiesPage.class); + } + public static void clickOnMysqlConnectionButton() { + ElementHelper.clickOnElement(MySQLPropertiesPage.connectorMysql); + } + + public static void clickOnMySQLConnection() { + String connectionName = PluginPropertyUtils.pluginProp("connection.name"); + ElementHelper.clickOnElement(MySQLPropertiesPage.mySQLConnection(connectionName)); + } +} diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/actions/package-info.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/actions/package-info.java new file mode 100644 index 000000000..09d9cf26e --- /dev/null +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/actions/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Package contains the actions for Mysql features. + */ +package io.cdap.plugin.mysql.actions; diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/locators/MySQLPropertiesPage.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/locators/MySQLPropertiesPage.java new file mode 100644 index 000000000..2bbb6c94d --- /dev/null +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/locators/MySQLPropertiesPage.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql.locators; + +import io.cdap.e2e.utils.SeleniumDriver; +import org.openqa.selenium.By; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.FindBy; +import org.openqa.selenium.support.How; + + +/** + * MySQL source - Properties page - Locators. + */ +public class MySQLPropertiesPage { + + @FindBy(how = How.XPATH, using = "//div[@data-cy='connector-MySQL']") + public static WebElement connectorMysql; + + public static WebElement mySQLConnection(String connectionName) { + return SeleniumDriver.getDriver().findElement( + By.xpath("//div[contains(text(),'" + connectionName + "')]")); + } +} diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/locators/package-info.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/locators/package-info.java new file mode 100644 index 000000000..72c31fec9 --- /dev/null +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/locators/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Package contains the locators for Mysql features. + */ +package io.cdap.plugin.mysql.locators; diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/package-info.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/package-info.java new file mode 100644 index 000000000..cbc3530ad --- /dev/null +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Package contains the BQValidation for Mysql. + */ +package io.cdap.plugin.mysql; diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunner.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunner.java index 8fc4f2a1b..f749c57c5 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunner.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunner.java @@ -26,7 +26,9 @@ @CucumberOptions( features = {"src/e2e-test/features"}, glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.mysql.stepsdesign"}, - tags = {"@Mysql"}, + tags = {"@Mysql and not @Plugin-20670"}, + /* TODO :Enable tests once issue fixed https://cdap.atlassian.net/browse/CDAP-20670 + */ plugin = {"pretty", "html:target/cucumber-html-report/mysql", "json:target/cucumber-reports/cucumber-mysql.json", "junit:target/cucumber-reports/cucumber-mysql.xml"} diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunnerRequired.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunnerRequired.java index fff521b31..77c142efa 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunnerRequired.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunnerRequired.java @@ -26,7 +26,9 @@ @CucumberOptions( features = {"src/e2e-test/features"}, glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.mysql.stepsdesign"}, - tags = {"@Mysql_Required"}, + tags = {"@Mysql_Required and not @Plugin-20670"}, + /* TODO :Enable tests once issue fixed https://cdap.atlassian.net/browse/CDAP-20670 + */ plugin = {"pretty", "html:target/cucumber-html-report/mysql-required", "json:target/cucumber-reports/cucumber-mysql-required.json", "junit:target/cucumber-reports/cucumber-mysql-required.xml"} diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/Mysql.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/Mysql.java index e8fdb5dd1..919ed8d38 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/Mysql.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/Mysql.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021 Cask Data, Inc. + * Copyright © 2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -16,18 +16,24 @@ package io.cdap.plugin.mysql.stepsdesign; +import io.cdap.e2e.pages.actions.CdfPipelineRunAction; +import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.CdfHelper; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.MysqlClient; +import io.cdap.plugin.mysql.BQValidation; +import io.cdap.plugin.mysql.actions.MySQLPropertiesPageActions; import io.cucumber.java.en.Then; import org.junit.Assert; import stepsdesign.BeforeActions; - +import java.io.IOException; import java.sql.SQLException; +import java.text.ParseException; /** * MYSQL Plugin related step design. */ + public class Mysql implements CdfHelper { @Then("Click on preview data for MySQL sink") @@ -37,14 +43,63 @@ public void clickOnPreviewDataForMysqlSink() { @Then("Validate the values of records transferred to target table is equal to the values from source table") public void validateTheValuesOfRecordsTransferredToTargetTableIsEqualToTheValuesFromSourceTable() throws - SQLException, ClassNotFoundException { + SQLException, ClassNotFoundException { int countRecords = MysqlClient.countRecord(PluginPropertyUtils.pluginProp("targetTable")); Assert.assertEquals("Number of records transferred should be equal to records out ", - countRecords, recordOut()); + countRecords, recordOut()); BeforeActions.scenario.write(" ******** Number of records transferred ********:" + countRecords); boolean recordsMatched = MysqlClient.validateRecordValues(PluginPropertyUtils.pluginProp("sourceTable"), - PluginPropertyUtils.pluginProp("targetTable")); + PluginPropertyUtils.pluginProp("targetTable")); + Assert.assertTrue("Value of records transferred to the target table should be equal to the value " + + "of the records in the source table", recordsMatched); + } + + @Then("Select Mysql Connection") + public void clickOnMysqlConnectionButton() { + MySQLPropertiesPageActions.clickOnMysqlConnectionButton(); + } + + @Then("Use new connection") + public void clickOnNewMySQLConnection() { + MySQLPropertiesPageActions.clickOnMySQLConnection(); + } + + @Then("Validate OUT record count is equal to records transferred to target BigQuery table") + public void validateOUTRecordCountIsEqualToRecordsTransferredToTargetBigQueryTable() + throws InterruptedException, IOException { + int targetBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqTargetTable")); + BeforeActions.scenario.write("No of Records Transferred to BigQuery:" + targetBQRecordsCount); + Assert.assertEquals("Out records should match with target BigQuery table records count", + CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), targetBQRecordsCount); + } + + @Then("Validate the values of records transferred to target Big Query table is equal to the values from source table") + public void validateTheValuesOfRecordsTransferredToTargetBigQueryTableIsEqualToTheValuesFromSourceTable() + throws InterruptedException, IOException, SQLException, ClassNotFoundException, ParseException { + int targetBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqTargetTable")); + BeforeActions.scenario.write("No of Records Transferred to BigQuery:" + targetBQRecordsCount); + Assert.assertEquals("Out records should match with target BigQuery table records count", + CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), targetBQRecordsCount); + + boolean recordsMatched = BQValidation.validateBQAndDBRecordValues( + PluginPropertyUtils.pluginProp("sourceTable"), + PluginPropertyUtils.pluginProp("bqTargetTable")); + Assert.assertTrue("Value of records transferred to the target table should be equal to the value " + + "of the records in the source table", recordsMatched); + } + @Then("Validate the values of records transferred to target MySQL table is equal to the values from source " + + "BigQuery table") + public void validateTheValuesOfRecordsTransferredToTargetMySQLTableIsEqualToTheValuesFromSourceBigQueryTable() + throws InterruptedException, IOException, SQLException, ClassNotFoundException, ParseException { + int sourceBQRecordsCount = BigQueryClient.countBqQuery(PluginPropertyUtils.pluginProp("bqSourceTable")); + BeforeActions.scenario.write("No of Records from source BigQuery table:" + sourceBQRecordsCount); + Assert.assertEquals("Out records should match with target MySQL table records count", + CdfPipelineRunAction.getCountDisplayedOnSourcePluginAsRecordsOut(), sourceBQRecordsCount); + + boolean recordsMatched = BQValidation.validateBQToDBRecordValues( + PluginPropertyUtils.pluginProp("bqSourceTable"), + PluginPropertyUtils.pluginProp("targetTable")); Assert.assertTrue("Value of records transferred to the target table should be equal to the value " + - "of the records in the source table", recordsMatched); + "of the records in the source table", recordsMatched); } -} + } diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/package-info.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/package-info.java index da490903b..56a575291 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/package-info.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/stepsdesign/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Cask Data, Inc. + * Copyright © 2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/package-info.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/package-info.java index 14fc395dd..3928d2033 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/package-info.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Cask Data, Inc. + * Copyright © 2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index 87c823507..0c1ac0400 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1 +1,13 @@ validationSuccessMessage=No errors found. +invalid.username.message=SQL error while getting query schema: +invalidtableName.error.message=Table '123#' does not exist. Ensure table '123#' is set correctly +invalidreferenceName.error.message=Invalid reference name +invalid.host.message=Exception while trying to validate schema of database table +invalid.password.message=SQL error while getting query schema: +invalid.databasename.message=SQL error while getting query schema: +invalid.query.message=SQL error while getting query schema: +numberofsplits.error.message=Unable to create config for batchsource Mysql 'numSplits' is invalid +boundingQuery.error.message=Bounding Query must be specified if Number of Splits is not set to 1. +splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 +invalid.sink.database.message=Exception while trying to validate schema +blank.username.message=Username is required when password is given. diff --git a/mysql-plugin/src/e2e-test/resources/pluginDataCyAttributes.properties b/mysql-plugin/src/e2e-test/resources/pluginDataCyAttributes.properties new file mode 100644 index 000000000..74a4271fb --- /dev/null +++ b/mysql-plugin/src/e2e-test/resources/pluginDataCyAttributes.properties @@ -0,0 +1,12 @@ +referenceName=referenceName +enableQuotedValues=switch-enableQuotedValues +inputsToLoadMemory=inMemoryInputs +projectId=project +datasetProjectId=datasetProject +dataset=dataset +table=table +truncateTable=switch-truncateTable +truncateTableMacroInput=truncateTable +updateTableSchema=switch-allowSchemaRelaxation +updateTableSchemaMacroInput=allowSchemaRelaxation +outputSchemaMacroInput=Output Schema-macro-input diff --git a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties index 9b428650f..215e4b830 100644 --- a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -1,12 +1,58 @@ driverName=mysql host=MYSQL_HOST username=MYSQL_USERNAME +invalid.username=root1 password=MYSQL_PASSWORD databaseName=sakila port=MYSQL_PORT +invalid.port=3301 +#advanced properties +numberofsplits=abc +numberofsplitsgenerate= +numbersplitsgenerate=2 +invalid.tablename=123# +invalid.referenceName=abc# +fetchsize=1000 +no.of.splits=1 +projectId=cdf-athena +#invalid properties +invalid.host=localhost1 +bqTargetTable=target-table +bqSourceTable=dummy +datasetprojectId=cdf-athena +dataset=test_automation +table.name=mysqltable +importQuery=select * from testTable where $CONDITIONS; +ImportQuery=where $CONDITIONS; +mysqlimportQuery= select * from SourceTable_AcGqvDqMqO where $CONDITIONS; +boundQuery=SELECT MIN(id) FROM sakila; +split.by.field.name=LastName +splitby=id +invalid.boundQuery=SELECT MIN(id),MAX(id) FROM table +targetTable=dummy +connectionArgumentsList=[{"key":"fetchsize","value":"1000"} +bqdatabase=Mysql_dataset +bqtable=Table1 + +query=select * from testTable; + +connection.name=dummy + +invalid.tableName=table@123 + + + sourceRef=source targetRef=target outputSchema=[{"key":"id","value":"int"},{"key":"lastName","value":"string"}] +invalid.query= select * +invalid.password=mysqlroot1 +invalid.database=test123 + + +#bq queries file path +CreateBQTableQueryFile=testData/BigQuery/BigQueryCreateTableQuery.txt +InsertBQDataQueryFile=testData/BigQuery/BigQueryInsertDataQuery.txt datatypesColumns=(ID varchar(100) PRIMARY KEY, COL1 bigint(20), COL2 bigint(20) unsigned, COL3 binary(1), \ COL4 bit(1), COL5 blob, COL6 tinyint(1), COL7 char(1), COL8 date, COL9 datetime, COL10 decimal(10,0), COL11 double, \ @@ -49,3 +95,7 @@ datatypesSchema=[{"key":"ID","value":"string"},{"key":"COL1","value":"long"},{"k {"key":"COL27","value":"timestamp"},{"key":"COL28","value":"bytes"},{"key":"COL29","value":"int"},\ {"key":"COL30","value":"int"},{"key":"COL31","value":"string"},{"key":"COL32","value":"bytes"},\ {"key":"COL33","value":"string"},{"key":"COL34","value":"int"}] + {"key":"COL33","value":"string"}] + +bqOutputDatatypesSchema=[{"key":"id","value":"long"},{"key":"lastName","value":"string"}] + diff --git a/mysql-plugin/src/e2e-test/resources/testData/BigQuery/BigQueryCreateTableQuery.txt b/mysql-plugin/src/e2e-test/resources/testData/BigQuery/BigQueryCreateTableQuery.txt new file mode 100644 index 000000000..5ab1bdd42 --- /dev/null +++ b/mysql-plugin/src/e2e-test/resources/testData/BigQuery/BigQueryCreateTableQuery.txt @@ -0,0 +1 @@ +create table `DATASET.TABLE_NAME` (id INT64,lastName STRING) diff --git a/mysql-plugin/src/e2e-test/resources/testData/BigQuery/BigQueryInsertDataQuery.txt b/mysql-plugin/src/e2e-test/resources/testData/BigQuery/BigQueryInsertDataQuery.txt new file mode 100644 index 000000000..2fc7552d5 --- /dev/null +++ b/mysql-plugin/src/e2e-test/resources/testData/BigQuery/BigQueryInsertDataQuery.txt @@ -0,0 +1,4 @@ +insert into `DATASET.TABLE_NAME` (id,lastName) values +(92233720,'BHATNAGAR'), +(92233721,'KUMAR'); + diff --git a/pom.xml b/pom.xml index 35a5f4161..b94cb1994 100644 --- a/pom.xml +++ b/pom.xml @@ -397,7 +397,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + 2.22.0 -Xmx3g -Djava.awt.headless=true -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC -Djava.net.preferIPv4Stack=true ${surefire.redirectTestOutputToFile} @@ -631,7 +631,14 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.0.0-M5 + 3.0.0 + + + org.apache.maven.surefire + surefire-junit47 + 3.0.0 + + ${TEST_RUNNER}