From 8475af7a0d82af8ceeffb5054d6295e8567678c6 Mon Sep 17 00:00:00 2001 From: haxiaolin Date: Thu, 26 Nov 2020 18:55:36 +0800 Subject: [PATCH 1/2] HBASE-25334 TestRSGroupsFallback.testFallback is flaky --- .../apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java index ea5e226b7f6b..b9dbbd01c846 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java @@ -25,12 +25,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RSGroupTests; import org.apache.hadoop.hbase.util.Bytes; @@ -61,6 +63,7 @@ public class TestRSGroupsFallback extends TestRSGroupsBase { public static void setUp() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean(RSGroupBasedLoadBalancer.FALLBACK_GROUP_ENABLE_KEY, true); + conf.setInt(HConstants.HBASE_BALANCER_MAX_BALANCING, 0); setUpTestBeforeClass(); MASTER.balanceSwitch(true); } @@ -118,7 +121,8 @@ public void testFallback() throws Exception { } private void assertRegionsInGroup(TableName table, String group) throws IOException { - TEST_UTIL.waitUntilAllRegionsAssigned(table); + ProcedureTestingUtility.waitAllProcedures( + TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor()); RSGroupInfo rsGroup = ADMIN.getRSGroup(group); MASTER.getAssignmentManager().getRegionStates().getRegionsOfTable(table).forEach(region -> { Address regionOnServer = MASTER.getAssignmentManager().getRegionStates() From d4d5a50617ada5d8fe0067302cf87c5d7e43a049 Mon Sep 17 00:00:00 2001 From: haxiaolin Date: Sun, 6 Dec 2020 16:59:02 +0800 Subject: [PATCH 2/2] fix testCrashProcedureReplay and testFallback --- .../org/apache/hadoop/hbase/master/ServerManager.java | 3 ++- .../org/apache/hadoop/hbase/master/TestDeadServer.java | 9 +++++++-- .../hadoop/hbase/rsgroup/TestRSGroupsFallback.java | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 9e666c56a890..8977174edba7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -501,7 +501,8 @@ public DeadServer getDeadServers() { * @return true if any RS are being processed as dead, false if not */ public boolean areDeadServersInProgress() throws IOException { - return master.getProcedures().stream().anyMatch(p -> p instanceof ServerCrashProcedure); + return master.getProcedures().stream() + .anyMatch(p -> !p.isFinished() && p instanceof ServerCrashProcedure); } void letRegionServersShutdown() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java index 39163ab1de08..b0d6cb63ea76 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java @@ -25,6 +25,7 @@ import java.util.Set; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; @@ -93,15 +94,19 @@ public static void tearDownAfterClass() throws Exception { } @Test - public void testCrashProcedureReplay() throws IOException { + public void testCrashProcedureReplay() throws Exception { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); final ProcedureExecutor pExecutor = master.getMasterProcedureExecutor(); ServerCrashProcedure proc = new ServerCrashProcedure( pExecutor.getEnvironment(), hostname123, false, false); + pExecutor.stop(); ProcedureTestingUtility.submitAndWait(pExecutor, proc); - assertTrue(master.getServerManager().areDeadServersInProgress()); + + ProcedureTestingUtility.restart(pExecutor); + ProcedureTestingUtility.waitProcedure(pExecutor, proc); + assertFalse(master.getServerManager().areDeadServersInProgress()); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java index b9dbbd01c846..478ffc654757 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java @@ -106,7 +106,7 @@ public void testFallback() throws Exception { // add a new server to default group, regions move to default group TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); - MASTER.balance(); + assertTrue(MASTER.balance()); assertRegionsInGroup(tableName, RSGroupInfo.DEFAULT_GROUP); // add a new server to test group, regions move back @@ -114,7 +114,7 @@ public void testFallback() throws Exception { TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000); ADMIN.moveServersToRSGroup( Collections.singleton(t.getRegionServer().getServerName().getAddress()), groupName); - MASTER.balance(); + assertTrue(MASTER.balance()); assertRegionsInGroup(tableName, groupName); TEST_UTIL.deleteTable(tableName);