From a984130bdf6af6144d77c4d0148faacd2b50c57e Mon Sep 17 00:00:00 2001 From: georgema-google Date: Tue, 25 Jul 2023 13:49:46 -0400 Subject: [PATCH 1/3] Fixed flakey tests in Bidirectional Replication endpoint tests --- ...eBidirectionalReplicationEndpointTest.java | 18 +++++++-------- ...eBidirectionalReplicationEndpointTest.java | 23 ++++++++----------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java index 9e50c56df7..c9e9600248 100644 --- a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java +++ b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; -import org.apache.hadoop.hbase.replication.ReplicationEndpoint.ReplicateContext; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.junit.After; import org.junit.AfterClass; @@ -60,12 +59,12 @@ public class HbaseToCloudBigtableBidirectionalReplicationEndpointTest { public static class TestReplicationEndpoint extends HbaseToCloudBigtableReplicationEndpoint { - static AtomicInteger replicatedEntries = new AtomicInteger(); + static AtomicInteger replicatingEntries = new AtomicInteger(); @Override public boolean replicate(ReplicateContext replicateContext) { boolean result = super.replicate(replicateContext); - replicatedEntries.getAndAdd(replicateContext.getEntries().size()); + replicatingEntries.getAndAdd(replicateContext.getEntries().size()); return result; } } @@ -141,7 +140,7 @@ public void setupTestCase() throws IOException { hbaseTable = hbaseConnection.getTable(table1); // Reset the entry counts for TestReplicationEndpoint - TestReplicationEndpoint.replicatedEntries.set(0); + TestReplicationEndpoint.replicatingEntries.set(0); } private void createTables(TableName tableName, int cf1Scope, int cf2Scope) throws IOException { @@ -185,15 +184,16 @@ public void testDropsReplicatedEntry() throws IOException, InterruptedException // Wait for replication TestUtils.waitForReplication( () -> { - // Only one entry should've been replicated - return TestReplicationEndpoint.replicatedEntries.get() >= 1; + // Both entries should've gone through the replication process + // However, one would've been filtered out during this process + return TestReplicationEndpoint.replicatingEntries.get() >= 2; }); // Hbase table should have both mutations - Assert.assertTrue(hbaseTable.get(new Get(TestUtils.ROW_KEY)).size() == 1); - Assert.assertTrue(hbaseTable.get(new Get(TestUtils.ROW_KEY_2)).size() == 1); + Assert.assertEquals(1, hbaseTable.get(new Get(TestUtils.ROW_KEY)).size()); + Assert.assertEquals(1, hbaseTable.get(new Get(TestUtils.ROW_KEY_2)).size()); // Cbt table should have only one mutation Assert.assertTrue(cbtTable.get(new Get(TestUtils.ROW_KEY)).isEmpty()); - Assert.assertTrue(cbtTable.get(new Get(TestUtils.ROW_KEY_2)).size() == 1); + Assert.assertEquals(1, cbtTable.get(new Get(TestUtils.ROW_KEY_2)).size()); } } diff --git a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java index df30199c5d..0b694d0438 100644 --- a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java +++ b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java @@ -61,12 +61,13 @@ public class HbaseToCloudBigtableBidirectionalReplicationEndpointTest { public static class TestReplicationEndpoint extends HbaseToCloudBigtableReplicationEndpoint { - static AtomicInteger replicatedEntries = new AtomicInteger(); + static AtomicInteger replicatingEntries = new AtomicInteger(); @Override public boolean replicate(ReplicateContext replicateContext) { boolean result = super.replicate(replicateContext); - replicatedEntries.getAndAdd(replicateContext.getEntries().size()); + replicatingEntries.getAndAdd(replicateContext.getEntries().size()); + return result; } } @@ -144,9 +145,7 @@ public void setupTestCase() throws IOException { hbaseTable = hbaseConnection.getTable(table1); // Reset the entry counts for TestReplicationEndpoint - HbaseToCloudBigtableBidirectionalReplicationEndpointTest.TestReplicationEndpoint - .replicatedEntries - .set(0); + TestReplicationEndpoint.replicatingEntries.set(0); } private void createTables(TableName tableName, int cf1Scope, int cf2Scope) throws IOException { @@ -191,18 +190,16 @@ public void testDropsReplicatedEntry() throws IOException, InterruptedException // Wait for replication TestUtils.waitForReplication( () -> { - // Only one entry should've been replicated - return HbaseToCloudBigtableBidirectionalReplicationEndpointTest.TestReplicationEndpoint - .replicatedEntries - .get() - >= 1; + // Both entries should've gone through the replication process + // However, one would've been filtered out during this process + return TestReplicationEndpoint.replicatingEntries.get() >= 2; }); // Hbase table should have both mutations - Assert.assertTrue(hbaseTable.get(new Get(TestUtils.ROW_KEY)).size() == 1); - Assert.assertTrue(hbaseTable.get(new Get(TestUtils.ROW_KEY_2)).size() == 1); + Assert.assertEquals(1, hbaseTable.get(new Get(TestUtils.ROW_KEY)).size()); + Assert.assertEquals(1, hbaseTable.get(new Get(TestUtils.ROW_KEY_2)).size()); // Cbt table should have only one mutation Assert.assertTrue(cbtTable.get(new Get(TestUtils.ROW_KEY)).isEmpty()); - Assert.assertTrue(cbtTable.get(new Get(TestUtils.ROW_KEY_2)).size() == 1); + Assert.assertEquals(1, cbtTable.get(new Get(TestUtils.ROW_KEY_2)).size()); } } From 8317a8f498b7ead3140939372de40a1a57cdb9c5 Mon Sep 17 00:00:00 2001 From: georgema-google Date: Tue, 25 Jul 2023 13:52:56 -0400 Subject: [PATCH 2/3] Fixed flakey tests in Bidirectional Replication endpoint tests --- ...HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java index c9e9600248..41dcb13f6f 100644 --- a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java +++ b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; +import org.apache.hadoop.hbase.replication.ReplicationEndpoint.ReplicateContext; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.junit.After; import org.junit.AfterClass; From 560d8674553ed26730123e2a2d364bbe1ee5161c Mon Sep 17 00:00:00 2001 From: georgema-google Date: Tue, 25 Jul 2023 14:00:57 -0400 Subject: [PATCH 3/3] Removed unused import --- ...HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java | 1 - ...HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java | 1 - 2 files changed, 2 deletions(-) diff --git a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java index 41dcb13f6f..c9e9600248 100644 --- a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java +++ b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-1.x-replication/src/test/java/com/google/cloud/bigtable/hbase1_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; -import org.apache.hadoop.hbase.replication.ReplicationEndpoint.ReplicateContext; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.junit.After; import org.junit.AfterClass; diff --git a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java index 0b694d0438..88c8d2c2a8 100644 --- a/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java +++ b/hbase-migration-tools/bigtable-hbase-replication/bigtable-hbase-2.x-replication/src/test/java/com/google/cloud/bigtable/hbase2_x/replication/HbaseToCloudBigtableBidirectionalReplicationEndpointTest.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; -import org.apache.hadoop.hbase.replication.ReplicationEndpoint.ReplicateContext; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.junit.After; import org.junit.AfterClass;