Bigtable E2E coverage as per ITN class.

.github/workflows/e2e.yml

@@ -40,7 +40,7 @@ jobs:
           )
     strategy:
       matrix:
-        tests: [bigquery, common, gcs, pubsub, spanner, gcscreate, gcsdelete, gcsmove, bigqueryexecute, gcscopy, datastore]
+        tests: [bigquery, common, gcs, pubsub, spanner, gcscreate, gcsdelete, gcsmove, bigqueryexecute, gcscopy, datastore, bigtable]
       fail-fast: false
     steps:
       # Pinned 1.0.0 version

pom.xml

@@ -854,6 +854,12 @@
       <version>0.2.0</version>
     </dependency>
     <!-- End: dependency used by the Dataplex connector -->
+    <!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-bigtable -->
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-bigtable</artifactId>
+      <version>1.17.1</version>
+    </dependency>
   </dependencies>
 
   <build>

src/e2e-test/features/bigtable/BigTableToBigTable.feature

@@ -0,0 +1,125 @@
+# Copyright © 2024 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+@BigTable @BIGTABLE_SOURCE_TEST
+Feature: BigTable source - Verification of BigTable to BigTable Successful Data Transfer
+
+  @BIGTABLE_SINK_TEST
+  Scenario: To verify data is getting transferred from BigTable source table to BigTable sink table
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Bigtable" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Bigtable" from the plugins list as: "Sink"
+    Then Connect plugins: "Bigtable" and "Bigtable2" to establish connection
+    Then Navigate to the properties page of plugin: "Bigtable"
+    Then Enter input plugin property: "referenceName" with value: "CBTSourceReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableInstance"
+    Then Enter input plugin property: "table" with value: "bigtableSourceTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsourceMappings"
+    Then Select Macro action of output schema property: "outputSchemaMacroInput" and set the value to "macroKeyString"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Bigtable2"
+    Then Enter input plugin property: "referenceName" with value: "CBTSinkReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableTargetInstance"
+    Then Enter input plugin property: "table" with value: "bigtableTargetTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsinkMappings"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "macroKeyString"
+    Then Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate OUT record count is equal to IN record count
+    Then Validate data transferred to target bigtable table with data of source bigtable table
+
+  @EXISTING_BIGTABLE_SINK
+  Scenario: To verify data is getting transferred from BigTable source table to existing BigTable sink
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Bigtable" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Bigtable" from the plugins list as: "Sink"
+    Then Connect plugins: "Bigtable" and "Bigtable2" to establish connection
+    Then Navigate to the properties page of plugin: "Bigtable"
+    Then Enter input plugin property: "referenceName" with value: "CBTSourceReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableInstance"
+    Then Enter input plugin property: "table" with value: "bigtableSourceTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsourceMappings"
+    Then Select Macro action of output schema property: "outputSchemaMacroInput" and set the value to "macroKeyString"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Bigtable2"
+    Then Enter input plugin property: "referenceName" with value: "CBTSinkReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableTargetInstance"
+    Then Enter input plugin property: "table" with value: "bigtableTargetExistingTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsinkMappings"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "macroKeyString"
+    Then Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate OUT record count is equal to IN record count
+    Then Validate data transferred to existing target bigtable table with data of source bigtable table
+
+  @BIGTABLE_SINK_TEST
+  Scenario: To verify data is getting transferred from unvalidated BigTable source table to BigTable sink table
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Bigtable" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Bigtable" from the plugins list as: "Sink"
+    Then Connect plugins: "Bigtable" and "Bigtable2" to establish connection
+    Then Navigate to the properties page of plugin: "Bigtable"
+    Then Enter input plugin property: "referenceName" with value: "CBTSourceReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableInstance"
+    Then Enter input plugin property: "table" with value: "bigtableSourceTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsourceMappings"
+    Then Select Macro action of output schema property: "outputSchemaMacroInput" and set the value to "macroKeyString"
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Bigtable2"
+    Then Enter input plugin property: "referenceName" with value: "CBTSinkReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableTargetInstance"
+    Then Enter input plugin property: "table" with value: "bigtableTargetTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsinkMappings"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "macroKeyString"
+    Then Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate OUT record count is equal to IN record count
+    Then Validate data transferred to target bigtable table with data of source bigtable table

src/e2e-test/features/spanner/source/SpannerToSpanner_withConnections.feature

@@ -1,5 +1,5 @@
 @Spanner_Source @SPANNER_TEST
-Feature: Spanner source - Verification of Spanner to Spanner successful data transfer using connections
+Feature: Spanner source - Verification Of Spanner to Spanner successful data transfer using connections
 
   @SPANNER_SOURCE_BASIC_TEST @SPANNER_SINK_TEST @SPANNER_CONNECTION @Spanner_Source_Required
   Scenario: To verify data transfer from Spanner to Spanner with pipeline connection created from wrangler

src/e2e-test/java/io/cdap/plugin/bigtable/runners/TestRunner.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright © 2024 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.plugin.bigtable.runners;
+
+import io.cucumber.junit.Cucumber;
+import io.cucumber.junit.CucumberOptions;
+import org.junit.runner.RunWith;
+
+/**
+ * Test Runner to execute Bigtable testcases.
+ */
+@RunWith(Cucumber.class)
+@CucumberOptions(
+        features = {"src/e2e-test/features"},
+        glue = {"io.cdap.plugin.bigtable.stepsdesign", "io.cdap.plugin.common.stepsdesign", "stepsdesign"},
+        tags = {"@BigTable"},
+        monochrome = true,
+        plugin = {"pretty", "html:target/cucumber-html-report/bigtable",
+                "json:target/cucumber-reports/cucumber-bigtable.json",
+                "junit:target/cucumber-reports/cucumber-bigtable.xml"}
+)
+public class TestRunner {
+}
+
+
+

src/e2e-test/java/io/cdap/plugin/bigtable/runners/package-info.java

@@ -0,0 +1,5 @@
+/**
+ * Package contains the test runner for the Cloud BigTable features.
+ */
+
+package io.cdap.plugin.bigtable.runners;

src/e2e-test/java/io/cdap/plugin/bigtable/stepsdesign/BigtableBase.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright © 2024 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.plugin.bigtable.stepsdesign;
+
+import io.cdap.e2e.utils.PluginPropertyUtils;
+import io.cdap.plugin.common.stepsdesign.TestSetupHooks;
+import io.cdap.plugin.utils.BigTableClient;
+import io.cucumber.java.en.Then;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
+
+import java.io.IOException;
+
+/**
+ * BigTable plugin related common test step definitions.
+ */
+
+public class BigtableBase {
+    @Then("Validate data transferred to target bigtable table with data of source bigtable table")
+    public void validateDataTransferToTargetBigtableTable() throws IOException {
+        Table sinkTable = BigTableClient.getTable(TestSetupHooks.bigTableConnection,
+                PluginPropertyUtils.pluginProp("bigtableTargetTable"));
+        validateData(sinkTable);
+    }
+    @Then("Validate data transferred to existing target bigtable table with data of source bigtable table")
+    public void validateDataTransferToTargetExistingBigtableTable() throws IOException {
+        Table existingSinkTable = BigTableClient.getTable(TestSetupHooks.bigTableExistingTargetTableConnection,
+                PluginPropertyUtils.pluginProp("bigtableTargetExistingTable"));
+        validateData(existingSinkTable);
+    }
+    public static void validateData(Table tableToBeValidated) throws IOException {
+        Result result = tableToBeValidated.get(new Get(Bytes.toBytes("r1")));
+        Assert.assertTrue(Bytes.toBoolean(result.getValue(Bytes.toBytes("cf1"), Bytes.toBytes("boolean_column"))));
+        Assert.assertEquals("bytes",
+                Bytes.toString(result.getValue(Bytes.toBytes("cf2"), Bytes.toBytes("bytes_column"))));
+        Assert.assertEquals(10.5D,
+                Bytes.toDouble(result.getValue(Bytes.toBytes("cf1"), Bytes.toBytes("double_column"))),
+                0.0000001);
+        Assert.assertEquals(10.5F,
+                Bytes.toFloat(result.getValue(Bytes.toBytes("cf2"), Bytes.toBytes("float_column"))),
+                0.0000001);
+        Assert.assertEquals(1,
+                Bytes.toInt(result.getValue(Bytes.toBytes("cf1"), Bytes.toBytes("int_column"))));
+        Assert.assertEquals(10L,
+                Bytes.toLong(result.getValue(Bytes.toBytes("cf2"), Bytes.toBytes("long_column"))));
+        Assert.assertEquals("string",
+                Bytes.toString(result.getValue(Bytes.toBytes("cf1"), Bytes.toBytes("string_column"))));
+
+    }
+}
+

src/e2e-test/java/io/cdap/plugin/bigtable/stepsdesign/package-info.java

@@ -0,0 +1,4 @@
+/**
+ * Package contains the base for the Cloud BigTable features.
+ */
+package io.cdap.plugin.bigtable.stepsdesign;