HBASE-26286: Add support for specifying store file tracker when resto…

…ring or cloning snapshot Signed-off-by: Duo Zhang <zhangduo@apache.org> Signed-off-by: Josh Elser <elserj@apache.org> Change-Id: I7b59b46cfcf66557f645ac8d108510201fc217e7
vinayakphegde · Jan 11, 2022 · 34a3645 · 34a3645
1 parent c316905
commit 34a3645
Show file tree

Hide file tree

Showing 23 changed files with 384 additions and 66 deletions.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -70,6 +70,7 @@
 import org.apache.yetus.audience.InterfaceAudience;
 
 import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.yetus.audience.InterfaceStability;
 
 /**
  * The administrative API for HBase. Obtain an instance from {@link Connection#getAdmin()} and
@@ -2388,7 +2389,25 @@ default void cloneSnapshot(byte[] snapshotName, TableName tableName)
    */
   default void cloneSnapshot(String snapshotName, TableName tableName)
       throws IOException, TableExistsException, RestoreSnapshotException {
-    cloneSnapshot(snapshotName, tableName, false);
+    cloneSnapshot(snapshotName, tableName, false, null);
+  }
+
+  /**
+   * Create a new table by cloning the snapshot content.
+   * @param snapshotName name of the snapshot to be cloned
+   * @param tableName name of the table where the snapshot will be restored
+   * @param restoreAcl <code>true</code> to clone acl into newly created table
+   * @param customSFT specify the StoreFileTracker used for the table
+   * @throws IOException if a remote or network exception occurs
+   * @throws TableExistsException if table to be created already exists
+   * @throws RestoreSnapshotException if snapshot failed to be cloned
+   * @throws IllegalArgumentException if the specified table has not a valid name
+   */
+  default void cloneSnapshot(String snapshotName, TableName tableName, boolean restoreAcl,
+    String customSFT)
+    throws IOException, TableExistsException, RestoreSnapshotException {
+    get(cloneSnapshotAsync(snapshotName, tableName, restoreAcl, customSFT), getSyncWaitTimeout(),
+      TimeUnit.MILLISECONDS);
   }
 
   /**
@@ -2435,8 +2454,25 @@ default Future<Void> cloneSnapshotAsync(String snapshotName, TableName tableName
    * @throws RestoreSnapshotException if snapshot failed to be cloned
    * @throws IllegalArgumentException if the specified table has not a valid name
    */
-  Future<Void> cloneSnapshotAsync(String snapshotName, TableName tableName, boolean restoreAcl)
-      throws IOException, TableExistsException, RestoreSnapshotException;
+  default Future<Void> cloneSnapshotAsync(String snapshotName, TableName tableName,
+    boolean restoreAcl)
+      throws IOException, TableExistsException, RestoreSnapshotException {
+    return cloneSnapshotAsync(snapshotName, tableName, restoreAcl, null);
+  }
+
+  /**
+   * Create a new table by cloning the snapshot content.
+   * @param snapshotName name of the snapshot to be cloned
+   * @param tableName name of the table where the snapshot will be restored
+   * @param restoreAcl <code>true</code> to clone acl into newly created table
+   * @param customSFT specify the StroreFileTracker used for the table
+   * @throws IOException if a remote or network exception occurs
+   * @throws TableExistsException if table to be created already exists
+   * @throws RestoreSnapshotException if snapshot failed to be cloned
+   * @throws IllegalArgumentException if the specified table has not a valid name
+   */
+  Future<Void> cloneSnapshotAsync(String snapshotName, TableName tableName, boolean restoreAcl,
+    String customSFT) throws IOException, TableExistsException, RestoreSnapshotException;
 
   /**
    * Execute a distributed procedure on a cluster.

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -872,8 +872,20 @@ default CompletableFuture<Void> cloneSnapshot(String snapshotName, TableName tab
    * @param tableName name of the table where the snapshot will be restored
    * @param restoreAcl <code>true</code> to restore acl of snapshot
    */
+  default CompletableFuture<Void> cloneSnapshot(String snapshotName, TableName tableName,
+      boolean restoreAcl) {
+    return cloneSnapshot(snapshotName, tableName, restoreAcl, null);
+  }
+
+  /**
+   * Create a new table by cloning the snapshot content.
+   * @param snapshotName name of the snapshot to be cloned
+   * @param tableName name of the table where the snapshot will be restored
+   * @param restoreAcl <code>true</code> to restore acl of snapshot
+   * @param customSFT specify the StroreFileTracker used for the table
+   */
   CompletableFuture<Void> cloneSnapshot(String snapshotName, TableName tableName,
-      boolean restoreAcl);
+      boolean restoreAcl, String customSFT);
 
   /**
    * List completed snapshots.

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -483,14 +483,14 @@ public CompletableFuture<Void> restoreSnapshot(String snapshotName) {
 
   @Override
   public CompletableFuture<Void> restoreSnapshot(String snapshotName, boolean takeFailSafeSnapshot,
-      boolean restoreAcl) {
+    boolean restoreAcl) {
     return wrap(rawAdmin.restoreSnapshot(snapshotName, takeFailSafeSnapshot, restoreAcl));
   }
 
   @Override
   public CompletableFuture<Void> cloneSnapshot(String snapshotName, TableName tableName,
-      boolean restoreAcl) {
-    return wrap(rawAdmin.cloneSnapshot(snapshotName, tableName, restoreAcl));
+      boolean restoreAcl, String customSFT) {
+    return wrap(rawAdmin.cloneSnapshot(snapshotName, tableName, restoreAcl, customSFT));
   }
 
   @Override

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.java
@@ -183,6 +183,11 @@ public interface ColumnFamilyDescriptor {
    * @return A clone value. Null if no mapping for the key
    */
   Bytes getValue(Bytes key);
+  /**
+   * @param key The key.
+   * @return A clone value. Null if no mapping for the key
+   */
+  String getValue(String key);
   /**
    * @param key The key.
    * @return A clone value. Null if no mapping for the key

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java
@@ -652,6 +652,12 @@ public byte[] getValue(byte[] key) {
       return value == null ? null : value.get();
     }
 
+    @Override
+    public String getValue(String key) {
+      Bytes rval = values.get(new Bytes(Bytes.toBytes(key)));
+      return rval == null ? null : Bytes.toString(rval.get(), rval.getOffset(), rval.getLength());
+    }
+
     @Override
     public Map<Bytes, Bytes> getValues() {
       return Collections.unmodifiableMap(values);

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -92,6 +92,27 @@
 import org.apache.hadoop.hbase.security.access.Permission;
 import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
 import org.apache.hadoop.hbase.security.access.UserPermission;
+import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
+import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
+import org.apache.hadoop.hbase.util.Addressing;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
+
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos;
@@ -2641,7 +2662,7 @@ public void restoreSnapshot(final String snapshotName, final boolean takeFailSaf
     try {
       // Restore snapshot
       get(
-        internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl),
+        internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl, null),
         syncWaitTimeout,
         TimeUnit.MILLISECONDS);
     } catch (IOException e) {
@@ -2650,7 +2671,7 @@ public void restoreSnapshot(final String snapshotName, final boolean takeFailSaf
       if (takeFailSafeSnapshot) {
         try {
           get(
-            internalRestoreSnapshotAsync(failSafeSnapshotSnapshotName, tableName, restoreAcl),
+            internalRestoreSnapshotAsync(failSafeSnapshotSnapshotName, tableName, restoreAcl, null),
             syncWaitTimeout,
             TimeUnit.MILLISECONDS);
           String msg = "Restore snapshot=" + snapshotName +
@@ -2693,16 +2714,17 @@ public Future<Void> restoreSnapshotAsync(final String snapshotName)
       throw new TableNotDisabledException(tableName);
     }
 
-    return internalRestoreSnapshotAsync(snapshotName, tableName, false);
+    return internalRestoreSnapshotAsync(snapshotName, tableName, false, null);
   }
 
   @Override
   public Future<Void> cloneSnapshotAsync(String snapshotName, TableName tableName,
-      boolean restoreAcl) throws IOException, TableExistsException, RestoreSnapshotException {
+    boolean restoreAcl, String customSFT)
+    throws IOException, TableExistsException, RestoreSnapshotException {
     if (tableExists(tableName)) {
       throw new TableExistsException(tableName);
     }
-    return internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl);
+    return internalRestoreSnapshotAsync(snapshotName, tableName, restoreAcl, customSFT);
   }
 
   @Override
@@ -2791,7 +2813,7 @@ protected IsProcedureDoneResponse rpcCall() throws Exception {
    * @throws IllegalArgumentException if the restore request is formatted incorrectly
    */
   private Future<Void> internalRestoreSnapshotAsync(final String snapshotName,
-      final TableName tableName, final boolean restoreAcl)
+      final TableName tableName, final boolean restoreAcl, String customSFT)
       throws IOException, RestoreSnapshotException {
     final SnapshotProtos.SnapshotDescription snapshot =
         SnapshotProtos.SnapshotDescription.newBuilder()
@@ -2806,13 +2828,15 @@ private Future<Void> internalRestoreSnapshotAsync(final String snapshotName,
           Long nonce = ng.newNonce();
       @Override
       protected RestoreSnapshotResponse rpcCall() throws Exception {
-        final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder()
+        final RestoreSnapshotRequest.Builder builder = RestoreSnapshotRequest.newBuilder()
             .setSnapshot(snapshot)
             .setNonceGroup(nonceGroup)
             .setNonce(nonce)
-            .setRestoreACL(restoreAcl)
-            .build();
-        return master.restoreSnapshot(getRpcController(), request);
+            .setRestoreACL(restoreAcl);
+        if (customSFT != null) {
+          builder.setCustomSFT(customSFT);
+        }
+        return master.restoreSnapshot(getRpcController(), builder.build());
       }
     });
 
@@ -4481,5 +4505,4 @@ protected Boolean rpcCall(int callTimeout) throws Exception {
       }
     });
   }
-
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -1937,7 +1937,7 @@ public CompletableFuture<Void> restoreSnapshot(String snapshotName, boolean take
         } else if (!exists) {
           // if table does not exist, then just clone snapshot into new table.
           completeConditionalOnFuture(future,
-            internalRestoreSnapshot(snapshotName, finalTableName, restoreAcl));
+            internalRestoreSnapshot(snapshotName, finalTableName, restoreAcl, null));
         } else {
           addListener(isTableDisabled(finalTableName), (disabled, err4) -> {
             if (err4 != null) {
@@ -1973,12 +1973,13 @@ private CompletableFuture<Void> restoreSnapshot(String snapshotName, TableName t
           future.completeExceptionally(err);
         } else {
           // Step.2 Restore snapshot
-          addListener(internalRestoreSnapshot(snapshotName, tableName, restoreAcl),
+          addListener(internalRestoreSnapshot(snapshotName, tableName, restoreAcl, null),
             (void2, err2) -> {
               if (err2 != null) {
                 // Step.3.a Something went wrong during the restore and try to rollback.
                 addListener(
-                  internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName, restoreAcl),
+                  internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName, restoreAcl,
+                    null),
                   (void3, err3) -> {
                     if (err3 != null) {
                       future.completeExceptionally(err3);
@@ -2008,7 +2009,7 @@ private CompletableFuture<Void> restoreSnapshot(String snapshotName, TableName t
       });
       return future;
     } else {
-      return internalRestoreSnapshot(snapshotName, tableName, restoreAcl);
+      return internalRestoreSnapshot(snapshotName, tableName, restoreAcl, null);
     }
   }
 
@@ -2025,7 +2026,7 @@ private <T> void completeConditionalOnFuture(CompletableFuture<T> dependentFutur
 
   @Override
   public CompletableFuture<Void> cloneSnapshot(String snapshotName, TableName tableName,
-      boolean restoreAcl) {
+      boolean restoreAcl, String customSFT) {
     CompletableFuture<Void> future = new CompletableFuture<>();
     addListener(tableExists(tableName), (exists, err) -> {
       if (err != null) {
@@ -2034,25 +2035,30 @@ public CompletableFuture<Void> cloneSnapshot(String snapshotName, TableName tabl
         future.completeExceptionally(new TableExistsException(tableName));
       } else {
         completeConditionalOnFuture(future,
-          internalRestoreSnapshot(snapshotName, tableName, restoreAcl));
+          internalRestoreSnapshot(snapshotName, tableName, restoreAcl, customSFT));
       }
     });
     return future;
   }
 
   private CompletableFuture<Void> internalRestoreSnapshot(String snapshotName, TableName tableName,
-      boolean restoreAcl) {
+      boolean restoreAcl, String customSFT) {
     SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder()
       .setName(snapshotName).setTable(tableName.getNameAsString()).build();
     try {
       ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
     } catch (IllegalArgumentException e) {
       return failedFuture(e);
     }
+    RestoreSnapshotRequest.Builder builder =
+      RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot).setNonceGroup(ng.getNonceGroup())
+        .setNonce(ng.newNonce()).setRestoreACL(restoreAcl);
+    if(customSFT != null){
+      builder.setCustomSFT(customSFT);
+    }
     return waitProcedureResult(this.<Long> newMasterCaller().action((controller, stub) -> this
       .<RestoreSnapshotRequest, RestoreSnapshotResponse, Long> call(controller, stub,
-        RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot).setNonceGroup(ng.getNonceGroup())
-          .setNonce(ng.newNonce()).setRestoreACL(restoreAcl).build(),
+        builder.build(),
         (s, c, req, done) -> s.restoreSnapshot(c, req, done), (resp) -> resp.getProcId()))
       .call());
   }

diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -458,6 +458,7 @@ message RestoreSnapshotRequest {
   optional uint64 nonce_group = 2 [default = 0];
   optional uint64 nonce = 3 [default = 0];
   optional bool restoreACL = 4 [default = false];
+  optional string customSFT = 5;
 }
 
 message RestoreSnapshotResponse {

diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -208,6 +208,7 @@ message CloneSnapshotStateData {
   repeated RegionInfo region_info = 4;
   repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 5;
   optional bool restore_acl = 6;
+  optional string customSFT = 7;
 }
 
 enum RestoreSnapshotState {

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2497,8 +2497,8 @@ public TableDescriptor get() throws IOException {
 
   }
 
-  public long restoreSnapshot(final SnapshotDescription snapshotDesc,
-      final long nonceGroup, final long nonce, final boolean restoreAcl) throws IOException {
+  public long restoreSnapshot(final SnapshotDescription snapshotDesc, final long nonceGroup,
+    final long nonce, final boolean restoreAcl, final String customSFT) throws IOException {
     checkInitialized();
     getSnapshotManager().checkSnapshotSupport();
 
@@ -2507,18 +2507,19 @@ public long restoreSnapshot(final SnapshotDescription snapshotDesc,
     getClusterSchema().getNamespace(dstTable.getNamespaceAsString());
 
     return MasterProcedureUtil.submitProcedure(
-        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
-      @Override
-      protected void run() throws IOException {
+      new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
+        @Override
+        protected void run() throws IOException {
           setProcId(
-            getSnapshotManager().restoreOrCloneSnapshot(snapshotDesc, getNonceKey(), restoreAcl));
-      }
+            getSnapshotManager().restoreOrCloneSnapshot(snapshotDesc, getNonceKey(), restoreAcl,
+              customSFT));
+        }
 
-      @Override
-      protected String getDescription() {
-        return "RestoreSnapshotProcedure";
-      }
-    });
+        @Override
+        protected String getDescription() {
+          return "RestoreSnapshotProcedure";
+        }
+      });
   }
 
   private void checkTableExists(final TableName tableName)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1531,7 +1531,7 @@ public RestoreSnapshotResponse restoreSnapshot(RpcController controller,
       RestoreSnapshotRequest request) throws ServiceException {
     try {
       long procId = master.restoreSnapshot(request.getSnapshot(), request.getNonceGroup(),
-        request.getNonce(), request.getRestoreACL());
+        request.getNonce(), request.getRestoreACL(), request.getCustomSFT());
       return RestoreSnapshotResponse.newBuilder().setProcId(procId).build();
     } catch (ForeignException e) {
       throw new ServiceException(e.getCause());