HBASE-25549 Provide lazy mode when modifying table to avoid RIT storm

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -1033,7 +1033,25 @@ default void modifyTable(TableDescriptor td) throws IOException {
    * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
    *         operation to complete
    */
-  Future<Void> modifyTableAsync(TableDescriptor td) throws IOException;
+  default Future<Void> modifyTableAsync(TableDescriptor td) throws IOException{
+    return modifyTableAsync(td, false);
+  }
+
+  /**
+   * Same as {@link #modifyTableAsync(TableDescriptor td)}. except
+   * {@code lazyMode} will control whether user lazy mode to modify a table
+   * @param td description of the table
+   * @param lazyMode When the lazy mode is enabled, the modification will not
+   *                 reopen any regions of the table so as to avoid RIT.
+   *                 A region would not aware of this modification till it reopened
+   *                 by another procedure(e.g. balance, move).
+   *                 Note that it temporarily lead to inconsistencies
+   *                 in the configuration of regions
+   * @throws IOException if a remote or network exception occurs
+   * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
+   *         operation to complete
+   */
+  Future<Void> modifyTableAsync(TableDescriptor td, boolean lazyMode) throws IOException;
 
   /**
    * Shuts down the HBase cluster.

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java

@@ -467,7 +467,12 @@ public Future<Void> splitRegionAsync(byte[] regionName, byte[] splitPoint) throw
 
   @Override
   public Future<Void> modifyTableAsync(TableDescriptor td) throws IOException {
-    return admin.modifyTable(td);
+    return modifyTableAsync(td, false);
+  }
+
+  @Override
+  public Future<Void> modifyTableAsync(TableDescriptor td, boolean lazyMode) throws IOException {
+    return admin.modifyTable(td, lazyMode);
   }
 
   @Override

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java

@@ -180,7 +180,21 @@ CompletableFuture<Void> createTable(TableDescriptor desc, byte[] startKey, byte[
    * Modify an existing table, more IRB friendly version.
    * @param desc modified description of the table
    */
-  CompletableFuture<Void> modifyTable(TableDescriptor desc);
+  default CompletableFuture<Void> modifyTable(TableDescriptor desc){
+    return modifyTable(desc, false);
+  }
+
+  /**
+   * Modify an existing table, more IRB friendly version.
+   * @param desc description of the table
+   * @param lazyMode When the lazy mode is enabled, the modification will not
+   *                 reopen any regions of the table so as to avoid RIT.
+   *                 A region would not aware of this modification till it reopened
+   *                 by another procedure(e.g. balance, move).
+   *                 Note that it temporarily lead to inconsistencies in the configuration of regions
+   */
+  CompletableFuture<Void> modifyTable(TableDescriptor desc, boolean lazyMode);
+
 
   /**
    * Deletes a table.

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java

@@ -143,7 +143,12 @@ public CompletableFuture<Void> createTable(TableDescriptor desc, byte[][] splitK
 
   @Override
   public CompletableFuture<Void> modifyTable(TableDescriptor desc) {
-    return wrap(rawAdmin.modifyTable(desc));
+    return modifyTable(desc, false);
+  }
+
+  @Override
+  public CompletableFuture<Void> modifyTable(TableDescriptor desc, boolean lazyMode) {
+    return wrap(rawAdmin.modifyTable(desc, lazyMode));
   }
 
   @Override

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java

@@ -656,9 +656,14 @@ private CompletableFuture<Void> createTable(TableName tableName, CreateTableRequ
 
   @Override
   public CompletableFuture<Void> modifyTable(TableDescriptor desc) {
+    return modifyTable(desc, false);
+  }
+
+  @Override
+  public CompletableFuture<Void> modifyTable(TableDescriptor desc, boolean lazyMode) {
     return this.<ModifyTableRequest, ModifyTableResponse> procedureCall(desc.getTableName(),
       RequestConverter.buildModifyTableRequest(desc.getTableName(), desc, ng.getNonceGroup(),
-        ng.newNonce()), (s, c, req, done) -> s.modifyTable(c, req, done),
+        ng.newNonce(), lazyMode), (s, c, req, done) -> s.modifyTable(c, req, done),
       (resp) -> resp.getProcId(), new ModifyTableProcedureBiConsumer(this, desc.getTableName()));
   }
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java

@@ -54,22 +54,37 @@ public interface TableDescriptor {
       if (result != 0) {
         return result;
       }
+      result = getColumnFamilyComparator(cfComparator).compare(lhs, rhs);
+      if (result != 0){
+        return result;
+      }
+      // punt on comparison for ordering, just calculate difference
+      return Integer.compare(lhs.getValues().hashCode(), rhs.getValues().hashCode());
+    };
+  }
+
+  /**
+   * This comparator only compare ColumnFamilyDescriptor between two tables
+   */
+  static Comparator<TableDescriptor>
+  getColumnFamilyComparator(
+    Comparator<ColumnFamilyDescriptor> cfComparator) {
+    return (TableDescriptor lhs, TableDescriptor rhs) -> {
       Collection<ColumnFamilyDescriptor> lhsFamilies = Arrays.asList(lhs.getColumnFamilies());
       Collection<ColumnFamilyDescriptor> rhsFamilies = Arrays.asList(rhs.getColumnFamilies());
-      result = Integer.compare(lhsFamilies.size(), rhsFamilies.size());
+      int result = Integer.compare(lhsFamilies.size(), rhsFamilies.size());
       if (result != 0) {
         return result;
       }
 
       for (Iterator<ColumnFamilyDescriptor> it = lhsFamilies.iterator(), it2 =
-          rhsFamilies.iterator(); it.hasNext();) {
+           rhsFamilies.iterator(); it.hasNext();) {
         result = cfComparator.compare(it.next(), it2.next());
         if (result != 0) {
           return result;
         }
       }
-      // punt on comparison for ordering, just calculate difference
-      return Integer.compare(lhs.getValues().hashCode(), rhs.getValues().hashCode());
+      return 0;
     };
   }
 

hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java

@@ -1203,15 +1203,17 @@ public static CreateTableRequest buildCreateTableRequest(
    * @return a ModifyTableRequest
    */
   public static ModifyTableRequest buildModifyTableRequest(
-      final TableName tableName,
-      final TableDescriptor tableDesc,
-      final long nonceGroup,
-      final long nonce) {
+    final TableName tableName,
+    final TableDescriptor tableDesc,
+    final long nonceGroup,
+    final long nonce,
+    final boolean lazyMode) {
     ModifyTableRequest.Builder builder = ModifyTableRequest.newBuilder();
     builder.setTableName(ProtobufUtil.toProtoTableName((tableName)));
     builder.setTableSchema(ProtobufUtil.toTableSchema(tableDesc));
     builder.setNonceGroup(nonceGroup);
     builder.setNonce(nonce);
+    builder.setLazyMode(lazyMode);
     return builder.build();
   }
 

hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto

@@ -194,6 +194,7 @@ message ModifyTableRequest {
   required TableSchema table_schema = 2;
   optional uint64 nonce_group = 3 [default = 0];
   optional uint64 nonce = 4 [default = 0];
+  optional bool lazy_mode = 5 [default = false];
 }
 
 message ModifyTableResponse {

hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto

@@ -82,6 +82,7 @@ message ModifyTableStateData {
   required TableSchema modified_table_schema = 3;
   required bool delete_column_family_in_modify = 4;
   optional bool should_check_descriptor = 5;
+  optional bool lazy_mode = 6;
 }
 
 enum TruncateTableState {

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -680,7 +680,8 @@ private void initializeZKBasedSystemTrackers()
       throws IOException, KeeperException, ReplicationException {
     this.balancer = new RSGroupBasedLoadBalancer();
     this.balancer.setConf(conf);
-    this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
+    this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper
+      , this);
     this.loadBalancerTracker.start();
 
     this.regionNormalizerManager =
@@ -2353,9 +2354,15 @@ protected String getDescription() {
     });
   }
 
+  private long modifyTable(final TableName tableName,
+    final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce,
+    final boolean lazyMode) throws IOException{
+    return modifyTable(tableName,newDescriptorGetter, nonceGroup, nonce, lazyMode, true);
+  }
+
   private long modifyTable(final TableName tableName,
       final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce,
-      final boolean shouldCheckDescriptor) throws IOException {
+      final boolean shouldCheckDescriptor, final boolean lazyMode) throws IOException {
     return MasterProcedureUtil
         .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
           @Override
@@ -2374,7 +2381,7 @@ protected void run() throws IOException {
             // checks. This will block only the beginning of the procedure. See HBASE-19953.
             ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
             submitProcedure(new ModifyTableProcedure(procedureExecutor.getEnvironment(),
-                newDescriptor, latch, oldDescriptor, shouldCheckDescriptor));
+                newDescriptor, latch, oldDescriptor, shouldCheckDescriptor, lazyMode));
             latch.await();
 
             getMaster().getMasterCoprocessorHost().postModifyTable(tableName, oldDescriptor,
@@ -2391,14 +2398,14 @@ protected String getDescription() {
 
   @Override
   public long modifyTable(final TableName tableName, final TableDescriptor newDescriptor,
-      final long nonceGroup, final long nonce) throws IOException {
+      final long nonceGroup, final long nonce, final boolean lazyMode) throws IOException {
     checkInitialized();
     return modifyTable(tableName, new TableDescriptorGetter() {
       @Override
       public TableDescriptor get() throws IOException {
         return newDescriptor;
       }
-    }, nonceGroup, nonce, false);
+    }, nonceGroup, nonce, false, lazyMode);
 
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java

@@ -1464,7 +1464,8 @@ public ModifyTableResponse modifyTable(RpcController controller,
         ProtobufUtil.toTableName(req.getTableName()),
         ProtobufUtil.toTableDescriptor(req.getTableSchema()),
         req.getNonceGroup(),
-        req.getNonce());
+        req.getNonce(),
+        req.getLazyMode());
       return ModifyTableResponse.newBuilder().setProcId(procId).build();
     } catch (IOException ioe) {
       throw new ServiceException(ioe);

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java

@@ -213,11 +213,31 @@ public long truncateTable(
    * @param nonce
    * @throws IOException
    */
+  default long modifyTable(
+    final TableName tableName,
+    final TableDescriptor descriptor,
+    final long nonceGroup,
+    final long nonce)
+    throws IOException{
+    return modifyTable(tableName, descriptor, nonceGroup, nonce, false);
+  }
+
+
+  /**
+   * Modify the descriptor of an existing table
+   * @param tableName The table name
+   * @param descriptor The updated table descriptor
+   * @param nonceGroup
+   * @param nonce
+   * @param lazyMode
+   * @throws IOException
+   */
   long modifyTable(
       final TableName tableName,
       final TableDescriptor descriptor,
       final long nonceGroup,
-      final long nonce)
+      final long nonce,
+      final boolean lazyMode)
       throws IOException;
 
   /**