From 5a29f8a3af7ccbb4fe277d3505953dc1c8e27dfa Mon Sep 17 00:00:00 2001 From: Jarryd Lee Date: Tue, 20 Dec 2022 16:32:53 -0800 Subject: [PATCH] HBASE-27541 Add support for defining restore hfile output path --- .../hadoop/hbase/backup/RestoreRequest.java | 15 +++++++++++++++ .../hbase/backup/impl/RestoreTablesClient.java | 10 +++++++--- .../mapreduce/MapReduceHFileSplitterJob.java | 1 + .../hadoop/hbase/backup/util/BackupUtils.java | 10 +++++++++- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java index eb4786f57869..4e097188fe7f 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java @@ -37,6 +37,11 @@ public Builder withBackupRootDir(String backupRootDir) { return this; } + public Builder withTargetRootDir(String targetRootDir) { + request.setTargetRootDir(targetRootDir); + return this; + } + public Builder withBackupId(String backupId) { request.setBackupId(backupId); return this; @@ -68,6 +73,7 @@ public RestoreRequest build() { } private String backupRootDir; + private String targetRootDir; private String backupId; private boolean check = false; private TableName[] fromTables; @@ -86,6 +92,15 @@ private RestoreRequest setBackupRootDir(String backupRootDir) { return this; } + public String getTargetRootDir() { + return targetRootDir; + } + + public RestoreRequest setTargetRootDir(String targetRootDir) { + this.targetRootDir = targetRootDir; + return this; + } + public String getBackupId() { return backupId; } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java index 9ec2442a3d93..b8bf1e13cedc 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreTablesClient.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.backup.HBackupFileSystem; import org.apache.hadoop.hbase.backup.RestoreRequest; import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; +import org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob; import org.apache.hadoop.hbase.backup.util.RestoreTool; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; @@ -55,11 +56,11 @@ public class RestoreTablesClient { private String backupId; private TableName[] sTableArray; private TableName[] tTableArray; - private String targetRootDir; + private String backupRootDir; private boolean isOverwrite; public RestoreTablesClient(Connection conn, RestoreRequest request) { - this.targetRootDir = request.getBackupRootDir(); + this.backupRootDir = request.getBackupRootDir(); this.backupId = request.getBackupId(); this.sTableArray = request.getFromTables(); this.tTableArray = request.getToTables(); @@ -69,6 +70,9 @@ public RestoreTablesClient(Connection conn, RestoreRequest request) { this.isOverwrite = request.isOverwrite(); this.conn = conn; this.conf = conn.getConfiguration(); + if (request.getTargetRootDir() != null) { + conf.set(MapReduceHFileSplitterJob.BULK_OUTPUT_ROOT_DIR, request.getTargetRootDir()); + } } /** @@ -249,7 +253,7 @@ public void execute() throws IOException { // case RESTORE_IMAGES: HashMap backupManifestMap = new HashMap<>(); // check and load backup image manifest for the tables - Path rootPath = new Path(targetRootDir); + Path rootPath = new Path(backupRootDir); HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath, backupId); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java index 766a99d778b8..41511086d2be 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java @@ -56,6 +56,7 @@ public class MapReduceHFileSplitterJob extends Configured implements Tool { private static final Logger LOG = LoggerFactory.getLogger(MapReduceHFileSplitterJob.class); final static String NAME = "HFileSplitterJob"; public final static String BULK_OUTPUT_CONF_KEY = "hfile.bulk.output"; + public static final String BULK_OUTPUT_ROOT_DIR = "hfile.bulk.output.root.dir"; public final static String TABLES_KEY = "hfile.input.tables"; public final static String TABLE_MAP_KEY = "hfile.input.tablesmap"; private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java index ef97b195e287..4f18f1b70875 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.backup.RestoreRequest; import org.apache.hadoop.hbase.backup.impl.BackupManifest; import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; +import org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionInfo; @@ -687,7 +688,14 @@ public static boolean validate(HashMap backupManifest public static Path getBulkOutputDir(String tableName, Configuration conf, boolean deleteOnExit) throws IOException { - FileSystem fs = FileSystem.get(conf); + FileSystem fs; + String bulkOutputRootDir = conf.get(MapReduceHFileSplitterJob.BULK_OUTPUT_ROOT_DIR); + if (bulkOutputRootDir != null) { + Path rootDir = new Path(bulkOutputRootDir); + fs = FileSystem.get(rootDir.toUri(), conf); + } else { + fs = FileSystem.get(conf); + } String tmp = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, fs.getHomeDirectory() + "/hbase-staging"); Path path = new Path(tmp + Path.SEPARATOR + "bulk_output-" + tableName + "-"