HBASE-21518 TestMasterFailoverWithProcedures is flaky

Signed-off-by: Sean Busbey <busbey@apache.org>
apache · Nov 30, 2018 · dfeab9f · dfeab9f
1 parent d42e0ad
commit dfeab9f
Showing 1 changed file with 17 additions and 7 deletions.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java
@@ -249,14 +249,24 @@ public static void shutdown(final List<MasterThread> masters,
       // Do backups first.
       JVMClusterUtil.MasterThread activeMaster = null;
       for (JVMClusterUtil.MasterThread t : masters) {
-        if (!t.master.isActiveMaster()) {
-          try {
-            t.master.stopMaster();
-          } catch (IOException e) {
-            LOG.error("Exception occurred while stopping master", e);
+        // Master was killed but could be still considered as active. Check first if it is stopped.
+        if (!t.master.isStopped()) {
+          if (!t.master.isActiveMaster()) {
+            try {
+              t.master.stopMaster();
+            } catch (IOException e) {
+              LOG.error("Exception occurred while stopping master", e);
+            }
+            LOG.info("Stopped backup Master {} is stopped: {}",
+                t.master.hashCode(), t.master.isStopped());
+          } else {
+            if (activeMaster != null) {
+              LOG.warn("Found more than 1 active master, hash {}", activeMaster.master.hashCode());
+            }
+            activeMaster = t;
+            LOG.debug("Found active master hash={}, stopped={}",
+                t.master.hashCode(), t.master.isStopped());
           }
-        } else {
-          activeMaster = t;
         }
       }
       // Do active after.