Skip to content

Commit

Permalink
[HUDI-66] FSUtils.getRelativePartitionPath does not handle repeated f…
Browse files Browse the repository at this point in the history
…older names
  • Loading branch information
milantracy authored and n3nash committed Apr 4, 2019
1 parent b34a204 commit d1d33f7
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,17 @@ public static List<String> getAllFoldersThreeLevelsDown(FileSystem fs, String ba
return datePartitions;
}

/**
* Given a base partition and a partition path, return
* relative path of partition path to the base path
*/
public static String getRelativePartitionPath(Path basePath, Path partitionPath) {
basePath = Path.getPathWithoutSchemeAndAuthority(basePath);
partitionPath = Path.getPathWithoutSchemeAndAuthority(partitionPath);
String partitionFullPath = partitionPath.toString();
int partitionStartIndex = partitionFullPath.lastIndexOf(basePath.getName());
int partitionStartIndex = partitionFullPath.indexOf(
basePath.getName(),
basePath.getParent() == null ? 0 : basePath.getParent().toString().length());
// Partition-Path could be empty for non-partitioned tables
return partitionStartIndex + basePath.getName().length() == partitionFullPath.length() ? "" :
partitionFullPath.substring(partitionStartIndex + basePath.getName().length() + 1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.Date;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.EnvironmentVariables;
Expand Down Expand Up @@ -92,4 +93,32 @@ public void testEnvVarVariablesPickedup() {
assertEquals("value11", conf.get("fs.key1"));
assertEquals("value2", conf.get("fs.key2"));
}

@Test
public void testGetRelativePartitionPath() {
Path basePath = new Path("/test/apache");
Path partitionPath = new Path("/test/apache/hudi/sub");
assertEquals("hudi/sub",FSUtils.getRelativePartitionPath(basePath, partitionPath));
}

@Test
public void testGetRelativePartitionPathSameFolder() {
Path basePath = new Path("/test");
Path partitionPath = new Path("/test");
assertEquals("", FSUtils.getRelativePartitionPath(basePath, partitionPath));
}

@Test
public void testGetRelativePartitionPathRepeatedFolderNameBasePath() {
Path basePath = new Path("/test/apache/apache");
Path partitionPath = new Path("/test/apache/apache/hudi");
assertEquals("hudi", FSUtils.getRelativePartitionPath(basePath, partitionPath));
}

@Test
public void testGetRelativePartitionPathRepeatedFolderNamePartitionPath() {
Path basePath = new Path("/test/apache");
Path partitionPath = new Path("/test/apache/apache/hudi");
assertEquals("apache/hudi", FSUtils.getRelativePartitionPath(basePath, partitionPath));
}
}

0 comments on commit d1d33f7

Please sign in to comment.