HBASE-21156 [hbck2] Queue an assign of hbase:meta and bulk assign/una…

…ssign

Adds 'raw' assigns and unassigns methods to Hbck Service.

Fixes HbckService so it works when cluster is Kerberized.

hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java

@@ -230,7 +230,6 @@
  * @see Admin
  */
 @InterfaceAudience.Private
-@InterfaceStability.Evolving
 public class HBaseAdmin implements Admin {
   private static final Logger LOG = LoggerFactory.getLogger(HBaseAdmin.class);
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java

@@ -18,25 +18,36 @@
 package org.apache.hadoop.hbase.client;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableStateResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.HbckService.BlockingInterface;
 
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 
 /**
  * Use {@link ClusterConnection#getHbck()} to obtain an instance of {@link Hbck} instead of
- * constructing
- * an HBaseHbck directly. This will be mostly used by hbck tool.
+ * constructing an HBaseHbck directly.
  *
  * <p>Connection should be an <i>unmanaged</i> connection obtained via
  * {@link ConnectionFactory#createConnection(Configuration)}.</p>
  *
+ * <p>NOTE: The methods in here can do damage to a cluster if applied in the wrong sequence or at
+ * the wrong time. Use with caution. For experts only. These methods are only for the
+ * extreme case where the cluster has been damaged or has achieved an inconsistent state because
+ * of some unforeseen circumstance or bug and requires manual intervention.
+ *
  * <p>An instance of this class is lightweight and not-thread safe. A new instance should be created
  * by each thread. Pooling or caching of the instance is not recommended.</p>
  *
@@ -75,10 +86,6 @@ public boolean isAborted() {
     return this.aborted;
   }
 
-  /**
-   * NOTE: This is a dangerous action, as existing running procedures for the table or regions
-   * which belong to the table may get confused.
-   */
   @Override
   public TableState setTableStateInMeta(TableState state) throws IOException {
     try {
@@ -87,9 +94,38 @@ public TableState setTableStateInMeta(TableState state) throws IOException {
           RequestConverter.buildSetTableStateInMetaRequest(state));
       return TableState.convert(state.getTableName(), response.getTableState());
     } catch (ServiceException se) {
-      LOG.debug("ServiceException while updating table state in meta. table={}, state={}",
-          state.getTableName(), state.getState());
+      LOG.debug("table={}, state={}", state.getTableName(), state.getState(), se);
       throw new IOException(se);
     }
   }
+
+  @Override
+  public List<Long> assigns(List<String> encodedRegionNames) throws IOException {
+    try {
+      MasterProtos.AssignsResponse response =
+          this.hbck.assigns(rpcControllerFactory.newController(),
+              RequestConverter.toAssignRegionsRequest(encodedRegionNames));
+      return response.getPidList();
+    } catch (ServiceException se) {
+      LOG.debug(toCommaDelimitedString(encodedRegionNames), se);
+      throw new IOException(se);
+    }
+  }
+
+  @Override
+  public List<Long> unassigns(List<String> encodedRegionNames) throws IOException {
+    try {
+      MasterProtos.UnassignsResponse response =
+          this.hbck.unassigns(rpcControllerFactory.newController(),
+              RequestConverter.toUnassignRegionsRequest(encodedRegionNames));
+      return response.getPidList();
+    } catch (ServiceException se) {
+      LOG.debug(toCommaDelimitedString(encodedRegionNames), se);
+      throw new IOException(se);
+    }
+  }
+
+  private static String toCommaDelimitedString(List<String> list) {
+    return list.stream().collect(Collectors.joining(", "));
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java

@@ -19,6 +19,8 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.List;
+
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -27,7 +29,12 @@
  * Hbck APIs for HBase. Obtain an instance from {@link ClusterConnection#getHbck()} and call
  * {@link #close()} when done.
  * <p>Hbck client APIs will be mostly used by hbck tool which in turn can be used by operators to
- * fix HBase and bringging it to consistent state.</p>
+ * fix HBase and bringing it to consistent state.</p>
+ *
+ * <p>NOTE: The methods in here can do damage to a cluster if applied in the wrong sequence or at
+ * the wrong time. Use with caution. For experts only. These methods are only for the
+ * extreme case where the cluster has been damaged or has achieved an inconsistent state because
+ * of some unforeseen circumstance or bug and requires manual intervention.
  *
  * @see ConnectionFactory
  * @see ClusterConnection
@@ -36,15 +43,36 @@
 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.HBCK)
 public interface Hbck extends Abortable, Closeable {
   /**
-   * Update table state in Meta only. No procedures are submitted to open/ assign or close/
-   * unassign regions of the table. This is useful only when some procedures/ actions are stuck
-   * beause of inconsistency between region and table states.
+   * Update table state in Meta only. No procedures are submitted to open/assign or
+   * close/unassign regions of the table.
    *
-   * NOTE: This is a dangerous action, as existing running procedures for the table or regions
+   * <p>>NOTE: This is a dangerous action, as existing running procedures for the table or regions
    * which belong to the table may get confused.
    *
    * @param state table state
    * @return previous state of the table in Meta
    */
   TableState setTableStateInMeta(TableState state) throws IOException;
+
+  /**
+   * Like {@link Admin#assign(byte[])} but 'raw' in that it can do more than one Region at a time
+   * -- good if many Regions to online -- and it will schedule the assigns even in the case where
+   * Master is initializing (as long as the ProcedureExecutor is up). Does NOT call Coprocessor
+   * hooks.
+   * @param encodedRegionNames Region encoded names; e.g. 1588230740 is the hard-coded encoding
+   *                           for hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an
+   *                           example of what a random user-space encoded Region name looks like.
+   */
+  List<Long> assigns(List<String> encodedRegionNames) throws IOException;
+
+  /**
+   * Like {@link Admin#unassign(byte[], boolean)} but 'raw' in that it can do more than one Region
+   * at a time -- good if many Regions to offline -- and it will schedule the assigns even in the
+   * case where Master is initializing (as long as the ProcedureExecutor is up). Does NOT call
+   * Coprocessor hooks.
+   * @param encodedRegionNames Region encoded names; e.g. 1588230740 is the hard-coded encoding
+   *                           for hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an
+   *                           example of what a random user-space encoded Region name looks like.
+   */
+  List<Long> unassigns(List<String> encodedRegionNames) throws IOException;
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecurityInfo.java

@@ -20,12 +20,13 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
 import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.TokenIdentifier.Kind;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos;
+import org.apache.yetus.audience.InterfaceAudience;
 
 /**
  * Maps RPC protocol interfaces to required configuration
@@ -37,13 +38,19 @@ public class SecurityInfo {
   // populate info for known services
   static {
     infos.put(AdminProtos.AdminService.getDescriptor().getName(),
-        new SecurityInfo("hbase.regionserver.kerberos.principal", Kind.HBASE_AUTH_TOKEN));
+        new SecurityInfo("hbase.regionserver.kerberos.principal",
+            Kind.HBASE_AUTH_TOKEN));
     infos.put(ClientProtos.ClientService.getDescriptor().getName(),
-        new SecurityInfo("hbase.regionserver.kerberos.principal", Kind.HBASE_AUTH_TOKEN));
+        new SecurityInfo("hbase.regionserver.kerberos.principal",
+            Kind.HBASE_AUTH_TOKEN));
     infos.put(MasterService.getDescriptor().getName(),
         new SecurityInfo("hbase.master.kerberos.principal", Kind.HBASE_AUTH_TOKEN));
     infos.put(RegionServerStatusProtos.RegionServerStatusService.getDescriptor().getName(),
         new SecurityInfo("hbase.master.kerberos.principal", Kind.HBASE_AUTH_TOKEN));
+    infos.put(MasterProtos.HbckService.getDescriptor().getName(),
+        new SecurityInfo("hbase.master.kerberos.principal", Kind.HBASE_AUTH_TOKEN));
+    // NOTE: IF ADDING A NEW SERVICE, BE SURE TO UPDATE HBasePolicyProvider ALSO ELSE
+    // new Service will not be found when all is Kerberized!!!!
   }
 
   /**

hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java

@@ -25,6 +25,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.apache.hadoop.hbase.CellScannable;
 import org.apache.hadoop.hbase.ClusterMetrics.Option;
@@ -1888,4 +1889,25 @@ private static List<HBaseProtos.ServerName> toProtoServerNames(List<ServerName>
     return TransitReplicationPeerSyncReplicationStateRequest.newBuilder().setPeerId(peerId)
         .setSyncReplicationState(ReplicationPeerConfigUtil.toSyncReplicationState(state)).build();
   }
+
+  // HBCK2
+  public static MasterProtos.AssignsRequest toAssignRegionsRequest(
+      List<String> encodedRegionNames) {
+    MasterProtos.AssignsRequest.Builder b = MasterProtos.AssignsRequest.newBuilder();
+    return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)).build();
+  }
+
+  public static MasterProtos.UnassignsRequest toUnassignRegionsRequest(
+      List<String> encodedRegionNames) {
+    MasterProtos.UnassignsRequest.Builder b =
+        MasterProtos.UnassignsRequest.newBuilder();
+    return b.addAllRegion(toEncodedRegionNameRegionSpecifiers(encodedRegionNames)).build();
+  }
+
+  private static List<RegionSpecifier> toEncodedRegionNameRegionSpecifiers(
+      List<String> encodedRegionNames) {
+    return encodedRegionNames.stream().
+        map(r -> buildRegionSpecifier(RegionSpecifierType.ENCODED_REGION_NAME, Bytes.toBytes(r))).
+        collect(Collectors.toList());
+  }
 }

hbase-protocol-shaded/src/main/protobuf/Master.proto

@@ -485,10 +485,6 @@ message GetTableStateResponse {
   required TableState table_state = 1;
 }
 
-message SetTableStateInMetaRequest {
-  required TableName table_name = 1;
-  required TableState table_state = 2;
-}
 
 message GetClusterStatusRequest {
   repeated Option options = 1;
@@ -997,8 +993,65 @@ service MasterService {
 
 }
 
+// HBCK Service definitions.
+
+message SetTableStateInMetaRequest {
+  required TableName table_name = 1;
+  required TableState table_state = 2;
+}
+
+/** Like Admin's AssignRegionRequest except it can
+ * take one or more Regions at a time.
+ */
+// NOTE: In hbck.proto, there is a define for
+// AssignRegionRequest -- singular 'Region'. This
+// is plural to convey it can carry more than one
+// Region at a time.
+message AssignsRequest {
+  repeated RegionSpecifier region = 1;
+}
+
+/** Like Admin's AssignRegionResponse except it can
+ * return one or more pids as result -- one per assign.
+ */
+message AssignsResponse {
+  repeated uint64 pid = 1;
+}
+
+/** Like Admin's UnassignRegionRequest except it can
+ * take one or more Regions at a time.
+ */
+message UnassignsRequest {
+  repeated RegionSpecifier region = 1;
+}
+
+/** Like Admin's UnassignRegionResponse except it can
+ * return one or more pids as result -- one per unassign.
+ */
+message UnassignsResponse {
+  repeated uint64 pid = 1;
+}
+
 service HbckService {
   /** Update state of the table in meta only*/
   rpc SetTableStateInMeta(SetTableStateInMetaRequest)
     returns(GetTableStateResponse);
+
+  /**
+   * Assign regions.
+   * Like Admin's assign but works even if the
+   * Master is initializing. Also allows bulk'ing up
+   * assigns rather than one region at a time.
+   */
+  rpc Assigns(AssignsRequest)
+    returns(AssignsResponse);
+
+  /**
+   * Unassign regions
+   * Like Admin's unssign but works even if the
+   * Master is initializing. Also allows bulk'ing up
+   * assigns rather than one region at a time.
+   */
+  rpc Unassigns(UnassignsRequest)
+    returns(UnassignsResponse);
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -1194,8 +1194,8 @@ private boolean isRegionOnline(RegionInfo ri) throws InterruptedException {
       // Page will talk about loss of edits, how to schedule at least the meta WAL recovery, and
       // then how to assign including how to break region lock if one held.
       LOG.warn("{} is NOT online; state={}; ServerCrashProcedures={}. Master startup cannot " +
-          "progress, in holding-pattern until region onlined; operator intervention required. " +
-          "Schedule an assign.", ri.getRegionNameAsString(), rs, optProc.isPresent());
+          "progress, in holding-pattern until region onlined.",
+          ri.getRegionNameAsString(), rs, optProc.isPresent());
       // Check once-a-minute.
       if (rc == null) {
         rc = new RetryCounterFactory(1000).create();