E2E test coverage for BQMT

src/e2e-test/features/bigquery/multitablesink/BigQueryMultiTableAdditional.feature

@@ -0,0 +1,218 @@
+# Copyright © 2024 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@BigQueryMultiTable_Sink
+Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable successful data transfer
+
+  @BQ_TWO_SOURCE_BQMT_TEST @BQ_DELETE_TABLES_TEST
+  Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two new tables
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable2"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery2" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
+    Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
+    Then Navigate to the properties page of plugin: "BigQuery Multi Table"
+    And Enter input plugin property: "referenceName" with value: "Reference"
+    And Replace input plugin property: "project" with value: "projectId"
+    And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
+    And Enter input plugin property: "dataset" with value: "dataset"
+    Then Override Service account details if set in environment variables
+    Then Click plugin property: "flexibleSchema"
+    Then Validate "BigQuery Multi Table" plugin properties
+    And Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
+
+  @BQ_SINGLE_SOURCE_BQMT_TEST @BQ_SINK_BQMT_TEST
+  Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in one table
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
+    Then Navigate to the properties page of plugin: "BigQuery Multi Table"
+    And Enter input plugin property: "referenceName" with value: "Reference"
+    And Replace input plugin property: "project" with value: "projectId"
+    And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
+    And Enter input plugin property: "dataset" with value: "dataset"
+    Then Override Service account details if set in environment variables
+    Then Click plugin property: "flexibleSchema"
+    Then Validate "BigQuery Multi Table" plugin properties
+    And Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal
+
+  @BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
+  Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable2"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery2" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
+    Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
+    Then Navigate to the properties page of plugin: "BigQuery Multi Table"
+    And Enter input plugin property: "referenceName" with value: "Reference"
+    And Replace input plugin property: "project" with value: "projectId"
+    And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
+    And Enter input plugin property: "dataset" with value: "dataset"
+    Then Override Service account details if set in environment variables
+    Then Click plugin property: "flexibleSchema"
+    Then Validate "BigQuery Multi Table" plugin properties
+    And Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
+
+  @BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
+  Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables using truncate
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable2"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery2" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
+    Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
+    Then Navigate to the properties page of plugin: "BigQuery Multi Table"
+    And Enter input plugin property: "referenceName" with value: "Reference"
+    And Replace input plugin property: "project" with value: "projectId"
+    And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
+    And Enter input plugin property: "dataset" with value: "dataset"
+    Then Override Service account details if set in environment variables
+    Then Click plugin property: "flexibleSchema"
+    Then Toggle BigQuery sink property truncateTable to true
+    Then Validate "BigQuery Multi Table" plugin properties
+    And Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
+
+  @BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
+  Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables after updating schema
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable2"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery2" plugin properties
+    Then Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "BigQuery Multi Table" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "BigQuery Multi Table" to establish connection
+    Then Connect plugins: "BigQuery2" and "BigQuery Multi Table" to establish connection
+    Then Navigate to the properties page of plugin: "BigQuery Multi Table"
+    And Enter input plugin property: "referenceName" with value: "Reference"
+    And Replace input plugin property: "project" with value: "projectId"
+    And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
+    And Enter input plugin property: "dataset" with value: "dataset"
+    Then Override Service account details if set in environment variables
+    Then Click plugin property: "flexibleSchema"
+    Then Select radio button plugin property: "updateSchema" with value: "true"
+    Then Validate "BigQuery Multi Table" plugin properties
+    And Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate data transferred from BigQuery To BigQueryMultiTable is equal

src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunner.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright © 2021 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package io.cdap.plugin.bigquery.runners.multitablerunner;
+
+import io.cucumber.junit.Cucumber;
+import io.cucumber.junit.CucumberOptions;
+import org.junit.runner.RunWith;
+
+/**
+ * Test Runner to execute BigQuery Multitable Sink plugin test cases.
+ */
+@RunWith(Cucumber.class)
+@CucumberOptions(
+  features = {"src/e2e-test/features"},
+  glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
+    "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
+  tags = {"@BigQueryMultiTable_Sink"},
+  monochrome = true,
+  plugin = {"pretty", "html:target/cucumber-html-report/bigquerymultitable-sink",
+    "json:target/cucumber-reports/cucumber-bigquerymultitable-sink.json",
+    "junit:target/cucumber-reports/cucumber-bigquerymultitable-sink.xml"}
+)
+public class TestRunner {
+}

src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/TestRunnerRequired.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright © 2021 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package io.cdap.plugin.bigquery.runners.multitablerunner;
+
+import io.cucumber.junit.Cucumber;
+import io.cucumber.junit.CucumberOptions;
+import org.junit.runner.RunWith;
+
+/**
+ * Test Runner to execute only required BigQuery Multitable Sink plugin test cases.
+ */
+@RunWith(Cucumber.class)
+@CucumberOptions(
+  features = {"src/e2e-test/features"},
+  glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
+    "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
+  tags = {"@BigQueryMultiTable_Sink_Required"},
+  monochrome = true,
+  plugin = {"pretty", "html:target/cucumber-html-report/bigquerymultitable-sink-required",
+    "json:target/cucumber-reports/cucumber-bigquerymultitable-sink-required.json",
+    "junit:target/cucumber-reports/cucumber-bigquerymultitable-sink-required.xml"}
+)
+public class TestRunnerRequired {
+}

src/e2e-test/java/io/cdap/plugin/bigquery/runners/multitablerunner/package-info.java

@@ -0,0 +1,4 @@
+/**
+ * Package contains the runner for the multitable plugin features.
+ */
+package io.cdap.plugin.bigquery.runners.multitablerunner;

src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright © 2024 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.plugin.bigquery.stepsdesign;
+
+import io.cdap.e2e.utils.PluginPropertyUtils;
+import io.cucumber.java.en.Then;
+import org.junit.Assert;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * BigQuery MultiTable related common stepDesigns.
+ */
+public class BigQueryMultiTable {
+  @Then("Validate data transferred from BigQuery To BigQueryMultiTable is equal")
+  public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
+    throws IOException, InterruptedException {
+    List<String> sourceTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqSourceTable"),
+                                        PluginPropertyUtils.pluginProp("bqSourceTable2"));
+
+    List<String> targetTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqTargetTable"),
+                                              PluginPropertyUtils.pluginProp("bqTargetTable2"));
+    boolean recordsMatched = BigQueryMultiTableValidation.validateBQToBigQueryMultiTable(sourceTables, targetTables);
+    Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
+                        "of the records in the source table", recordsMatched);
+  }
+
+  @Then("Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal")
+  public void validateDataTransferredFromBigQueryToBigQueryMultiTableInOneTableIsEqual()
+    throws IOException, InterruptedException {
+    boolean recordsMatched = BigQueryMultiTableValidation.
+      validateBQToBigQueryMultiTable(Collections.singletonList(PluginPropertyUtils.pluginProp("bqSourceTable")),
+                                     Collections.singletonList(PluginPropertyUtils.pluginProp("bqTargetTable")));
+    Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
+                        "of the records in the source table", recordsMatched);
+  }
+}