Skip to content

Commit

Permalink
HIVE-23539 : Optimize data copy during repl load operation for HDFS b…
Browse files Browse the repository at this point in the history
…ased staging location - Incremental - Single Move - exim
  • Loading branch information
Pravin Sinha committed Jun 23, 2020
1 parent cfbeb48 commit a8fb7ec
Showing 1 changed file with 10 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,8 @@ private static Task<?> loadTable(URI fromURI, Table table, boolean replace, Path
boolean isAutoPurge = false;
boolean needRecycle = false;
boolean copyToMigratedTxnTable = replicationSpec.isMigratingToTxnTable();
boolean performOnlyMove = replicationSpec.isInReplicationScope() && Utils.onSameHDFSFileSystem(dataPath, tgtPath);
boolean performOnlyMove = performOnlyMoveOperation(replicationSpec, dataPath, tgtPath);
x.getLOG().debug("Perform only move operation:" + performOnlyMove);
if (replicationSpec.isInReplicationScope() && (copyToMigratedTxnTable ||
x.getCtx().getConf().getBoolean(REPL_ENABLE_MOVE_OPTIMIZATION.varname, false))) {
lft = performOnlyMove ? getLoadFileType(replicationSpec) : LoadFileType.IGNORE;
Expand Down Expand Up @@ -541,6 +542,11 @@ private static Task<?> loadTable(URI fromURI, Table table, boolean replace, Path
return loadTableTask;
}

private static boolean performOnlyMoveOperation(ReplicationSpec replSpec, Path dataPath, Path tgtPath) {
boolean exIm = ReplicationSpec.Type.IMPORT.equals(replSpec.getReplSpecType());
return !exIm && replSpec.isInReplicationScope() && Utils.onSameHDFSFileSystem(dataPath, tgtPath);
}

private static Task<?> createTableTask(ImportTableDesc tableDesc, EximUtil.SemanticAnalyzerWrapperContext x) {
return tableDesc.getCreateTableTask(x.getInputs(), x.getOutputs(), x.getConf());
}
Expand Down Expand Up @@ -617,7 +623,9 @@ private static Task<?> addSinglePartition(ImportTableDesc tblDesc,
*/
}
fixLocationInPartSpec(tblDesc, table, wh, replicationSpec, partSpec, x);
performOnlyMove = Utils.onSameHDFSFileSystem(new Path(srcLocation), new Path(partSpec.getLocation()));
performOnlyMove = performOnlyMoveOperation(replicationSpec, new Path(srcLocation),
new Path(partSpec.getLocation()));
x.getLOG().debug("Perform only move operation:" + performOnlyMove);
x.getLOG().debug("adding dependent CopyWork/AddPart/MoveWork for partition "
+ partSpecToString(partSpec.getPartSpec())
+ " with source location: " + srcLocation);
Expand Down

0 comments on commit a8fb7ec

Please sign in to comment.