From 95c66515c40b139b53cfb16a0cb6b4fd0d2f4f44 Mon Sep 17 00:00:00 2001 From: koodin9 Date: Mon, 17 Nov 2025 15:45:05 +0900 Subject: [PATCH 1/3] Fix invalid path error in ViewFs with linkMergeSlash --- .../src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 466057feec718..51b5b937a06f4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -719,7 +719,7 @@ protected InodeTree(final Configuration config, final String viewName, if (isMergeSlashConfigured) { Preconditions.checkNotNull(mergeSlashTarget); - root = new INodeLink(mountTableName, ugi, + root = new INodeLink("/", ugi, initAndGetTargetFs(), mergeSlashTarget); mountPoints.add(new MountPoint("/", (INodeLink) root)); rootFallbackLink = null; From f0dab4ea6322ce3fa8e30f5a1078f69f091e3115 Mon Sep 17 00:00:00 2001 From: koodin9 Date: Mon, 17 Nov 2025 16:15:22 +0900 Subject: [PATCH 2/3] add test --- .../TestViewFileSystemLinkMergeSlash.java | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java index a2d9c46215ce8..bf04739a19deb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java @@ -28,7 +28,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -228,4 +230,136 @@ public void testChildFileSystems() throws Exception { assertEquals(DistributedFileSystem.class, childFs[0].getClass(), "Unexpected child filesystem!"); } + + @Test + public void testListStatusReturnsCorrectPaths() throws Exception { + String clusterName = "ClusterLMS1"; + URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName, + "/", null, null); + + Configuration conf = new Configuration(); + ConfigUtil.addLinkMergeSlash(conf, clusterName, + fsDefault.getUri()); + + FileSystem vfs = FileSystem.get(viewFsUri, conf); + + // Create test directory structure + Path testDir = new Path("/testListStatus"); + Path subDir1 = new Path(testDir, "subdir1"); + Path subDir2 = new Path(testDir, "subdir2"); + Path file1 = new Path(testDir, "file1.txt"); + + try { + fsDefault.mkdirs(subDir1); + fsDefault.mkdirs(subDir2); + fsDefault.create(file1).close(); + + // Test listStatus returns correct ViewFS paths + FileStatus[] statuses = vfs.listStatus(testDir); + Assert.assertEquals("Should have 3 items", 3, statuses.length); + + for (FileStatus status : statuses) { + Path path = status.getPath(); + // Path should be viewfs:// URI, not hdfs:// + Assert.assertEquals("Scheme should be viewfs", + FsConstants.VIEWFS_SCHEME, path.toUri().getScheme()); + Assert.assertEquals("Authority should match cluster name", + clusterName, path.toUri().getAuthority()); + // Path should start with /testListStatus, not contain cluster name + Assert.assertTrue("Path should be absolute and correct: " + path, + path.toString().startsWith("viewfs://" + clusterName + "/testListStatus/")); + Assert.assertFalse("Path should not contain duplicate cluster name: " + path, + path.toString().contains(clusterName + "/" + clusterName)); + } + } finally { + fsDefault.delete(testDir, true); + vfs.close(); + } + } + + @Test + public void testListLocatedStatusReturnsCorrectPaths() throws Exception { + String clusterName = "ClusterLMS1"; + URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName, + "/", null, null); + + Configuration conf = new Configuration(); + ConfigUtil.addLinkMergeSlash(conf, clusterName, + fsDefault.getUri()); + + FileSystem vfs = FileSystem.get(viewFsUri, conf); + + // Create test directory structure + Path testDir = new Path("/testListLocatedStatus"); + Path file1 = new Path(testDir, "file1.txt"); + Path file2 = new Path(testDir, "file2.txt"); + + try { + fsDefault.mkdirs(testDir); + fsDefault.create(file1).close(); + fsDefault.create(file2).close(); + + // Test listLocatedStatus returns correct ViewFS paths + RemoteIterator iter = vfs.listLocatedStatus(testDir); + int count = 0; + while (iter.hasNext()) { + LocatedFileStatus status = iter.next(); + Path path = status.getPath(); + count++; + + // Path should be viewfs:// URI + Assert.assertEquals("Scheme should be viewfs", + FsConstants.VIEWFS_SCHEME, path.toUri().getScheme()); + Assert.assertEquals("Authority should match cluster name", + clusterName, path.toUri().getAuthority()); + // Path should be absolute and not contain relative path issues + Assert.assertTrue("Path should start correctly: " + path, + path.toString().startsWith("viewfs://" + clusterName + "/testListLocatedStatus/")); + Assert.assertFalse("Path should not contain duplicate cluster name: " + path, + path.toString().contains(clusterName + "/" + clusterName)); + } + Assert.assertEquals("Should have 2 files", 2, count); + } finally { + fsDefault.delete(testDir, true); + vfs.close(); + } + } + + @Test + public void testResolvedPathIsAbsolute() throws Exception { + String clusterName = "ClusterLMS1"; + URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName, + "/", null, null); + + Configuration conf = new Configuration(); + ConfigUtil.addLinkMergeSlash(conf, clusterName, + fsDefault.getUri()); + + FileSystem vfs = FileSystem.get(viewFsUri, conf); + + // Create nested directory structure + Path baseDir = new Path("/user/history/done"); + Path yearDir = new Path(baseDir, "2021"); + + try { + fsDefault.mkdirs(yearDir); + + // This is the exact scenario that caused the bug + FileStatus[] statuses = vfs.listStatus(baseDir); + Assert.assertEquals("Should have 1 directory", 1, statuses.length); + + Path resultPath = statuses[0].getPath(); + String expectedPath = "viewfs://" + clusterName + "/user/history/done/2021"; + Assert.assertEquals("Path should be correctly formed", + expectedPath, resultPath.toString()); + + // Verify path is absolute (starts with /) + Assert.assertTrue("Path should be absolute", + resultPath.toUri().getPath().startsWith("/")); + } finally { + fsDefault.delete(new Path("/user"), true); + vfs.close(); + } + } } + From b5428d0f5e3597d49555072bd51dc0ea7fe20984 Mon Sep 17 00:00:00 2001 From: koodin9 Date: Wed, 26 Nov 2025 13:14:03 +0900 Subject: [PATCH 3/3] fix test --- .../TestViewFileSystemLinkMergeSlash.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java index bf04739a19deb..55dbb9a855caa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkMergeSlash.java @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -256,20 +257,20 @@ public void testListStatusReturnsCorrectPaths() throws Exception { // Test listStatus returns correct ViewFS paths FileStatus[] statuses = vfs.listStatus(testDir); - Assert.assertEquals("Should have 3 items", 3, statuses.length); + assertEquals(3, statuses.length, "Should have 3 items"); for (FileStatus status : statuses) { Path path = status.getPath(); // Path should be viewfs:// URI, not hdfs:// - Assert.assertEquals("Scheme should be viewfs", - FsConstants.VIEWFS_SCHEME, path.toUri().getScheme()); - Assert.assertEquals("Authority should match cluster name", - clusterName, path.toUri().getAuthority()); + assertEquals(FsConstants.VIEWFS_SCHEME, path.toUri().getScheme(), + "Scheme should be viewfs"); + assertEquals(clusterName, path.toUri().getAuthority(), + "Authority should match cluster name"); // Path should start with /testListStatus, not contain cluster name - Assert.assertTrue("Path should be absolute and correct: " + path, - path.toString().startsWith("viewfs://" + clusterName + "/testListStatus/")); - Assert.assertFalse("Path should not contain duplicate cluster name: " + path, - path.toString().contains(clusterName + "/" + clusterName)); + assertTrue(path.toString().startsWith("viewfs://" + clusterName + "/testListStatus/"), + "Path should be absolute and correct: " + path); + assertFalse(path.toString().contains(clusterName + "/" + clusterName), + "Path should not contain duplicate cluster name: " + path); } } finally { fsDefault.delete(testDir, true); @@ -308,17 +309,17 @@ public void testListLocatedStatusReturnsCorrectPaths() throws Exception { count++; // Path should be viewfs:// URI - Assert.assertEquals("Scheme should be viewfs", - FsConstants.VIEWFS_SCHEME, path.toUri().getScheme()); - Assert.assertEquals("Authority should match cluster name", - clusterName, path.toUri().getAuthority()); + assertEquals(FsConstants.VIEWFS_SCHEME, path.toUri().getScheme(), + "Scheme should be viewfs"); + assertEquals(clusterName, path.toUri().getAuthority(), + "Authority should match cluster name"); // Path should be absolute and not contain relative path issues - Assert.assertTrue("Path should start correctly: " + path, - path.toString().startsWith("viewfs://" + clusterName + "/testListLocatedStatus/")); - Assert.assertFalse("Path should not contain duplicate cluster name: " + path, - path.toString().contains(clusterName + "/" + clusterName)); + assertTrue(path.toString().startsWith("viewfs://" + clusterName + "/testListLocatedStatus/"), + "Path should start correctly: " + path); + assertFalse(path.toString().contains(clusterName + "/" + clusterName), + "Path should not contain duplicate cluster name: " + path); } - Assert.assertEquals("Should have 2 files", 2, count); + assertEquals(2, count, "Should have 2 files"); } finally { fsDefault.delete(testDir, true); vfs.close(); @@ -346,16 +347,16 @@ public void testResolvedPathIsAbsolute() throws Exception { // This is the exact scenario that caused the bug FileStatus[] statuses = vfs.listStatus(baseDir); - Assert.assertEquals("Should have 1 directory", 1, statuses.length); + assertEquals(1, statuses.length, "Should have 1 directory"); Path resultPath = statuses[0].getPath(); String expectedPath = "viewfs://" + clusterName + "/user/history/done/2021"; - Assert.assertEquals("Path should be correctly formed", - expectedPath, resultPath.toString()); + assertEquals(expectedPath, resultPath.toString(), + "Path should be correctly formed"); // Verify path is absolute (starts with /) - Assert.assertTrue("Path should be absolute", - resultPath.toUri().getPath().startsWith("/")); + assertTrue(resultPath.toUri().getPath().startsWith("/"), + "Path should be absolute"); } finally { fsDefault.delete(new Path("/user"), true); vfs.close();