Skip to content

Commit

Permalink
java-sdk: refactor distcp checksum patch
Browse files Browse the repository at this point in the history
  • Loading branch information
tangyoupeng committed Jan 26, 2021
1 parent 1f672f1 commit 15db788
Showing 1 changed file with 4 additions and 24 deletions.
28 changes: 4 additions & 24 deletions sdk/java/src/main/java/com/juicefs/JuiceFileSystem.java
Expand Up @@ -63,30 +63,10 @@ public class JuiceFileSystem extends FilterFileSystem {
private synchronized static void patchDistCpChecksum() {
if (distcpPatched)
return;
ClassPool classPool = ClassPool.getDefault();
try {
String clsName = "org.apache.hadoop.tools.mapred.RetriableFileCopyCommand";
CtClass copyClass = classPool.get(clsName);
CtMethod method = copyClass.getDeclaredMethod("compareCheckSums");
method.insertBefore(
"if (sourceFS.getFileStatus(source).getBlockSize() != targetFS.getFileStatus(target).getBlockSize()) {return ;}");
byte[] bytecode = copyClass.toBytecode();
ClassDefinition definition = new ClassDefinition(Class.forName(clsName), bytecode);
RedefineClassAgent.redefineClasses(definition);
copyClass.detach();
} catch (NotFoundException | ClassNotFoundException e) {
} catch (NoClassDefFoundError e) {
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
if (stackTrace.length > 1) {
StackTraceElement element = stackTrace[stackTrace.length - 1];
if (element.getClassName().contains("DistCp")) {
LOG.warn(
"Please add tools.jar to classpath to skip checksum check for files with different block sizes.");
}
}
} catch (Throwable e) {
LOG.warn("patch distcp failed!", e);
}
PatchUtil.patchBefore("org.apache.hadoop.tools.mapred.RetriableFileCopyCommand",
"compareCheckSums",
null,
"if (sourceFS.getFileStatus(source).getBlockSize() != targetFS.getFileStatus(target).getBlockSize()) {return ;}");
distcpPatched = true;
}

Expand Down

0 comments on commit 15db788

Please sign in to comment.