HBASE-25656 Backport to branch-2.2: [HBASE-25548 Optionally allow sna…

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -1982,6 +1982,53 @@ void snapshot(String snapshotName,
       SnapshotType type) throws IOException, SnapshotCreationException,
       IllegalArgumentException;
 
+  /**
+   * Create typed snapshot of the table. Snapshots are considered unique based on <b>the name of the
+   * snapshot</b>. Snapshots are taken sequentially even when requested concurrently, across
+   * all tables. Attempts to take a snapshot with the same name (even a different type or with
+   * different parameters) will fail with a {@link SnapshotCreationException} indicating the
+   * duplicate naming. Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * Snapshot can live with ttl seconds.
+   *
+   * @param snapshotName  name to give the snapshot on the filesystem. Must be unique from all other
+   *                      snapshots stored on the cluster
+   * @param tableName     name of the table to snapshot
+   * @param type          type of snapshot to take
+   * @param snapshotProps snapshot additional properties e.g. TTL
+   * @throws IOException               we fail to reach the master
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException  if the snapshot request is formatted incorrectly
+   */
+  default void snapshot(String snapshotName, TableName tableName, SnapshotType type,
+    Map<String, Object> snapshotProps) throws IOException,
+    SnapshotCreationException, IllegalArgumentException {
+    snapshot(new SnapshotDescription(snapshotName, tableName, type, snapshotProps));
+  }
+
+  /**
+   * Create typed snapshot of the table. Snapshots are considered unique based on <b>the name of the
+   * snapshot</b>. Snapshots are taken sequentially even when requested concurrently, across
+   * all tables. Attempts to take a snapshot with the same name (even a different type or with
+   * different parameters) will fail with a {@link SnapshotCreationException} indicating the
+   * duplicate naming. Snapshot names follow the same naming constraints as tables in HBase. See
+   * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * Snapshot can live with ttl seconds.
+   *
+   * @param snapshotName  name to give the snapshot on the filesystem. Must be unique from all other
+   *                      snapshots stored on the cluster
+   * @param tableName     name of the table to snapshot
+   * @param snapshotProps snapshot additional properties e.g. TTL
+   * @throws IOException               we fail to reach the master
+   * @throws SnapshotCreationException if snapshot creation failed
+   * @throws IllegalArgumentException  if the snapshot request is formatted incorrectly
+   */
+  default void snapshot(String snapshotName, TableName tableName,
+    Map<String, Object> snapshotProps) throws IOException,
+    SnapshotCreationException, IllegalArgumentException {
+    snapshot(new SnapshotDescription(snapshotName, tableName, SnapshotType.FLUSH, snapshotProps));
+  }
+
   /**
    * Take a snapshot and wait for the server to complete that snapshot (blocking). Only a single
    * snapshot should be taken at a time for an instance of HBase, or results may be undefined (you

hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java

@@ -18,8 +18,11 @@
 package org.apache.hadoop.hbase.client;
 
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
 import org.apache.yetus.audience.InterfaceAudience;
 
+import java.util.Map;
+
 /**
  * The POJO equivalent of HBaseProtos.SnapshotDescription
  */
@@ -32,6 +35,8 @@ public class SnapshotDescription {
   private final long creationTime;
   private final int version;
 
+  private final long maxFileSize;
+
   public SnapshotDescription(String name) {
     this(name, (TableName)null);
   }
@@ -48,7 +53,7 @@ public SnapshotDescription(String name, String table) {
   }
 
   public SnapshotDescription(String name, TableName table) {
-    this(name, table, SnapshotType.DISABLED, null);
+    this(name, table, SnapshotType.DISABLED, null, -1, -1, null);
   }
 
   /**
@@ -63,7 +68,7 @@ public SnapshotDescription(String name, String table, SnapshotType type) {
   }
 
   public SnapshotDescription(String name, TableName table, SnapshotType type) {
-    this(name, table, type, null);
+    this(name, table, type, null, -1, -1, null);
   }
 
   /**
@@ -78,7 +83,7 @@ public SnapshotDescription(String name, String table, SnapshotType type, String
   }
 
   public SnapshotDescription(String name, TableName table, SnapshotType type, String owner) {
-    this(name, table, type, owner, -1, -1);
+    this(name, table, type, owner, -1, -1, null);
   }
 
   /**
@@ -90,17 +95,37 @@ public SnapshotDescription(String name, TableName table, SnapshotType type, Stri
   @Deprecated
   public SnapshotDescription(String name, String table, SnapshotType type, String owner,
       long creationTime, int version) {
-    this(name, TableName.valueOf(table), type, owner, creationTime, version);
+    this(name, TableName.valueOf(table), type, owner, creationTime, version, null);
   }
 
   public SnapshotDescription(String name, TableName table, SnapshotType type, String owner,
-      long creationTime, int version) {
+      long creationTime, int version, Map<String, Object> snapshotProps) {
     this.name = name;
     this.table = table;
     this.snapShotType = type;
     this.owner = owner;
     this.creationTime = creationTime;
     this.version = version;
+    this.maxFileSize = getLongFromSnapshotProps(snapshotProps, TableDescriptorBuilder.MAX_FILESIZE);
+  }
+
+  private long getLongFromSnapshotProps(Map<String, Object> snapshotProps, String property) {
+    return MapUtils.getLongValue(snapshotProps, property, -1);
+  }
+
+
+
+  /**
+   * SnapshotDescription Parameterized Constructor
+   *
+   * @param snapshotName  Name of the snapshot
+   * @param tableName     TableName associated with the snapshot
+   * @param type          Type of the snapshot - enum SnapshotType
+   * @param snapshotProps Additional properties for snapshot e.g. TTL
+   */
+  public SnapshotDescription(String snapshotName, TableName tableName, SnapshotType type,
+                             Map<String, Object> snapshotProps) {
+    this(snapshotName, tableName, type, null, -1, -1, snapshotProps);
   }
 
   public String getName() {
@@ -143,11 +168,14 @@ public int getVersion() {
     return this.version;
   }
 
+  public long getMaxFileSize() { return maxFileSize; }
+
   @Override
   public String toString() {
     return "SnapshotDescription: name = " + ((name != null) ? name : null) + "/table = "
         + ((table != null) ? table : null) + " /owner = " + ((owner != null) ? owner : null)
         + (creationTime != -1 ? ("/creationtime = " + creationTime) : "")
-        + (version != -1 ? ("/version = " + version) : "");
+        + (version != -1 ? ("/version = " + version) : "")
+        + (maxFileSize != -1 ? ("/maxFileSize = " + maxFileSize) : "");
   }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java

@@ -29,6 +29,7 @@
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -2925,6 +2926,9 @@ public static SnapshotType createSnapshotType(SnapshotProtos.SnapshotDescription
     if (snapshotDesc.getVersion() != -1) {
       builder.setVersion(snapshotDesc.getVersion());
     }
+    if (snapshotDesc.getMaxFileSize() != -1) {
+      builder.setMaxFileSize(snapshotDesc.getMaxFileSize());
+    }
     builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType()));
     SnapshotProtos.SnapshotDescription snapshot = builder.build();
     return snapshot;
@@ -2939,8 +2943,10 @@ public static SnapshotType createSnapshotType(SnapshotProtos.SnapshotDescription
    */
   public static SnapshotDescription
       createSnapshotDesc(SnapshotProtos.SnapshotDescription snapshotDesc) {
+    final Map<String, Object> snapshotProps = new HashMap<>();
+    snapshotProps.put(TableDescriptorBuilder.MAX_FILESIZE, snapshotDesc.getMaxFileSize());
     return new SnapshotDescription(snapshotDesc.getName(),
-        snapshotDesc.hasTable() ? TableName.valueOf(snapshotDesc.getTable()) : null,
+        snapshotDesc.hasTable() ? snapshotDesc.getTable() : null,
         createSnapshotType(snapshotDesc.getType()), snapshotDesc.getOwner(),
         snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
   }

hbase-protocol-shaded/src/main/protobuf/Snapshot.proto

@@ -44,6 +44,7 @@ message SnapshotDescription {
   optional int32 version = 5;
   optional string owner = 6;
   optional UsersAndPermissions users_and_permissions = 7;
+  optional int64 max_file_size = 8 [default = 0];
 }
 
 message SnapshotFileInfo {

hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java

@@ -149,6 +149,10 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
   /** number of current operations running on the master */
   public static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
 
+  /** Conf key for preserving original max file size configs */
+  public static final String SNAPSHOT_MAX_FILE_SIZE_PRESERVE =
+    "hbase.snapshot.max.filesize.preserve";
+
   private boolean stopped;
   private MasterServices master;  // Needed by TableEventHandlers
   private ProcedureCoordinator coordinator;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java

@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hbase.master.snapshot;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashSet;
@@ -33,6 +32,7 @@
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
@@ -141,12 +141,17 @@ public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices ma
   }
 
   private TableDescriptor loadTableDescriptor()
-      throws FileNotFoundException, IOException {
+      throws IOException {
     TableDescriptor htd =
       this.master.getTableDescriptors().get(snapshotTable);
     if (htd == null) {
       throw new IOException("TableDescriptor missing for " + snapshotTable);
     }
+    if (htd.getMaxFileSize()==-1 &&
+        this.snapshot.getMaxFileSize()>0) {
+      htd = TableDescriptorBuilder.newBuilder(htd).setValue(TableDescriptorBuilder.MAX_FILESIZE,
+        Long.toString(this.snapshot.getMaxFileSize())).build();
+    }
     return htd;
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java

@@ -227,7 +227,7 @@ public void testOfflineTableSnapshot() throws Exception {
     final String SNAPSHOT_NAME = "offlineTableSnapshot";
     byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
 
-    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME,
+    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME.getNameAsString(),
         SnapshotType.DISABLED, null, -1, SnapshotManifestV1.DESCRIPTOR_VERSION));
     LOG.debug("Snapshot completed.");
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestTakeSnapshotHandler.java

@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+
+/**
+ * Unfortunately, couldn't test TakeSnapshotHandler using mocks, because it relies on TableLock,
+ * which is tightly coupled to LockManager and LockProcedure classes, which are both final and
+ * prevents us from mocking its behaviour. Looks like an overkill having to emulate a
+ * whole cluster run for such a small optional property behaviour.
+ */
+@Category({ MediumTests.class})
+public class TestTakeSnapshotHandler {
+
+  private static HBaseTestingUtility UTIL;
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+    HBaseClassTestRule.forClass(TestTakeSnapshotHandler.class);
+
+  @Rule
+  public TestName name = new TestName();
+
+
+  @Before
+  public void setup()  {
+    UTIL = new HBaseTestingUtility();
+  }
+
+  public TableDescriptor createTableInsertDataAndTakeSnapshot(Map<String, Object> snapshotProps)
+      throws Exception {
+    TableDescriptor descriptor =
+      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
+        .setColumnFamily(
+          ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f")).build()).build();
+    UTIL.getConnection().getAdmin().createTable(descriptor);
+    Table table = UTIL.getConnection().getTable(descriptor.getTableName());
+    Put put = new Put(Bytes.toBytes("1"));
+    put.addColumn(Bytes.toBytes("f"), Bytes.toBytes("1"), Bytes.toBytes("v1"));
+    table.put(put);
+    String snapName = "snap"+name.getMethodName();
+    UTIL.getAdmin().snapshot(snapName, descriptor.getTableName(), snapshotProps);
+    TableName cloned = TableName.valueOf(name.getMethodName() + "clone");
+    UTIL.getAdmin().cloneSnapshot(snapName, cloned);
+    return descriptor;
+  }
+
+  @Test
+  public void testPreparePreserveMaxFileSizeEnabled() throws Exception {
+    UTIL.startMiniCluster();
+    Map<String, Object> snapshotProps = new HashMap<>();
+    snapshotProps.put(TableDescriptorBuilder.MAX_FILESIZE, Long.parseLong("21474836480"));
+    TableDescriptor descriptor = createTableInsertDataAndTakeSnapshot(snapshotProps);
+    TableName cloned = TableName.valueOf(name.getMethodName() + "clone");
+    assertEquals(-1,
+      UTIL.getAdmin().getDescriptor(descriptor.getTableName()).getMaxFileSize());
+    assertEquals(21474836480L, UTIL.getAdmin().getDescriptor(cloned).getMaxFileSize());
+  }
+
+  @Test
+  public void testPreparePreserveMaxFileSizeDisabled() throws Exception {
+    UTIL.startMiniCluster();
+    TableDescriptor descriptor = createTableInsertDataAndTakeSnapshot(null);
+    TableName cloned = TableName.valueOf(name.getMethodName() + "clone");
+    assertEquals(-1,
+      UTIL.getAdmin().getDescriptor(descriptor.getTableName()).getMaxFileSize());
+    assertEquals(-1, UTIL.getAdmin().getDescriptor(cloned).getMaxFileSize());
+  }
+
+  @After
+  public void shutdown() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+}

hbase-shell/src/main/ruby/hbase/admin.rb

@@ -1063,11 +1063,15 @@ def snapshot(table, snapshot_name, *args)
         @admin.snapshot(snapshot_name, table_name)
       else
         args.each do |arg|
+          snapshot_props = java.util.HashMap.new
+          max_filesize = arg[MAX_FILESIZE]
+          max_filesize = max_filesize ? max_filesize.to_java(:long) : -1
+          snapshot_props.put("MAX_FILESIZE", max_filesize)
           if arg[SKIP_FLUSH] == true
             @admin.snapshot(snapshot_name, table_name,
-                            org.apache.hadoop.hbase.client.SnapshotType::SKIPFLUSH)
+                            org.apache.hadoop.hbase.client.SnapshotType::SKIPFLUSH, snapshot_props)
           else
-            @admin.snapshot(snapshot_name, table_name)
+            @admin.snapshot(snapshot_name, table_name, snapshot_props)
           end
         end
       end

hbase-shell/src/main/ruby/shell/commands/snapshot.rb

@@ -24,7 +24,7 @@ def help
 Take a snapshot of specified table. Examples:
 
   hbase> snapshot 'sourceTable', 'snapshotName'
-  hbase> snapshot 'namespace:sourceTable', 'snapshotName', {SKIP_FLUSH => true}
+  hbase> snapshot 'namespace:sourceTable', 'snapshotName', {SKIP_FLUSH => true, MAX_FILESIZE => 21474836480}
 EOF
       end