HBASE-24117 Shutdown AssignmentManager before ProcedureExecutor may c…

…ause SCP to accidentally skip assigning a region (apache#1865)

Author: Duo Zhang
Reason: Bug
Ref: CDPD-15964

Signed-off-by: Michael Stack <stack@apache.org>

Change-Id: I6bb4b16b68ea8c9e5baa39aa719447268da24c5c
(cherry picked from commit 3419317)

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -1458,6 +1458,11 @@ protected void stopServiceThreads() {
 
     LOG.debug("Stopping service threads");
 
+    // stop procedure executor prior to other services such as server manager and assignment
+    // manager, as these services are important for some running procedures. See HBASE-24117 for
+    // example.
+    stopProcedureExecutor();
+
     if (this.quotaManager != null) {
       this.quotaManager.stop();
     }
@@ -1472,8 +1477,6 @@ protected void stopServiceThreads() {
       this.assignmentManager.stop();
     }
 
-    stopProcedureExecutor();
-
     if (this.walManager != null) {
       this.walManager.stop();
     }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java

@@ -24,6 +24,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
@@ -473,6 +474,12 @@ private void assignRegions(MasterProcedureEnv env, List<RegionInfo> regions) thr
         // updated the region location, or even finished itself, so the region is no longer on us
         // any more, we should not try to assign it again. Please see HBASE-23594 for more details.
         if (!serverName.equals(regionNode.getRegionLocation())) {
+          // See HBASE-24117, though we have already changed the shutdown order, it is still worth
+          // double checking here to confirm that we do not skip assignment incorrectly.
+          if (!am.isRunning()) {
+            throw new DoNotRetryIOException(
+              "AssignmentManager has been stopped, can not process assignment any more");
+          }
           LOG.info("{} found a region {} which is no longer on us {}, give up assigning...", this,
             regionNode, serverName);
           continue;