From 55b6e33020ddf4b8fd1a1909d39efb69c2b4653e Mon Sep 17 00:00:00 2001 From: Balaji Varadarajan Date: Thu, 12 Nov 2020 00:12:16 -0800 Subject: [PATCH] [HUDI-1395] HoodieSnapshotCopier to work correctly on non-partitioned datasets --- .../org/apache/hudi/utilities/HoodieSnapshotCopier.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java index 05b46273001c0..dd46815d09ed2 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java @@ -110,8 +110,8 @@ public void snapshot(JavaSparkContext jsc, String baseDir, final String outputDi dataFiles.forEach(hoodieDataFile -> filePaths.add(new Tuple2<>(partition, hoodieDataFile.getPath()))); // also need to copy over partition metadata - Path partitionMetaFile = - new Path(new Path(baseDir, partition), HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE); + Path partitionMetaFile = new Path(FSUtils.getPartitionPath(baseDir, partition), + HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE); if (fs1.exists(partitionMetaFile)) { filePaths.add(new Tuple2<>(partition, partitionMetaFile.toString())); } @@ -122,7 +122,7 @@ public void snapshot(JavaSparkContext jsc, String baseDir, final String outputDi context.foreach(filesToCopy, tuple -> { String partition = tuple._1(); Path sourceFilePath = new Path(tuple._2()); - Path toPartitionPath = new Path(outputDir, partition); + Path toPartitionPath = FSUtils.getPartitionPath(outputDir, partition); FileSystem ifs = FSUtils.getFs(baseDir, serConf.newCopy()); if (!ifs.exists(toPartitionPath)) {