HIVE-23539 : Optimize data copy during repl load operation for HDFS b…

…ased staging location
apache · Jun 15, 2020 · 5c2fb99 · 5c2fb99
1 parent 625d471
commit 5c2fb99
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@ -269,7 +269,7 @@ private Task<?> tasksForAddPartition(Table table, AlterTableAddPartitionDesc add
     }
 
     Path loadTmpDir = replicaWarehousePartitionLocation;
-    boolean performOnlyMove = Utils.onSameHDFSFileSystem(event.dataPath(), replicaWarehousePartitionLocation);
+
     // if move optimization is enabled, copy the files directly to the target path. No need to create the staging dir.
     LoadFileType loadFileType;
     if (event.replicationSpec().isInReplicationScope() &&
@@ -293,6 +293,8 @@ private Task<?> tasksForAddPartition(Table table, AlterTableAddPartitionDesc add
      * If the Repl staging directory ('hive.repl.rootdir') is on the target cluster itself and the FS scheme is hdfs,
      * data is moved directly from Repl staging data dir of partition to the partition's location on target warehouse.
      */
+    boolean performOnlyMove = event.replicationSpec().isInReplicationScope()
+            && Utils.onSameHDFSFileSystem(event.dataPath(), replicaWarehousePartitionLocation);
     Path moveSource = performOnlyMove ? partDataSrc : loadTmpDir;
     Task<?> movePartitionTask = null;
     if (loadFileType != LoadFileType.IGNORE) {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@ -276,7 +276,6 @@ private Task<?> loadTableTask(Table table, ReplicationSpec replicationSpec, Path
       Path fromURI) {
     Path dataPath = fromURI;
     Path loadTmpDir = tgtPath;
-    boolean performOnlyMove = Utils.onSameHDFSFileSystem(dataPath, tgtPath);
 
     // if move optimization is enabled, copy the files directly to the target path. No need to create the staging dir.
     LoadFileType loadFileType;
@@ -303,6 +302,7 @@ private Task<?> loadTableTask(Table table, ReplicationSpec replicationSpec, Path
      * data is moved directly from Repl staging data dir of the partition to the partition's location on target
      * warehouse.
      */
+    boolean performOnlyMove = replicationSpec.isInReplicationScope() && Utils.onSameHDFSFileSystem(dataPath, tgtPath);
     Path moveSrcPath = performOnlyMove ? dataPath : loadTmpDir;
 
     MoveWork moveWork = new MoveWork(new HashSet<>(), new HashSet<>(), null, null, false);