From 05f77fd2a53c0f0d3806adaa0ffbde88354ff4a9 Mon Sep 17 00:00:00 2001 From: Abhishek Das Date: Tue, 13 Jul 2021 12:47:43 -0700 Subject: [PATCH] HADOOP-17028. ViewFS should initialize mounted target filesystems lazily. Contributed by Abhishek Das (#2260) (cherry picked from commit 1dd03cc4b573270dc960117c3b6c74bb78215caa) --- .../apache/hadoop/fs/viewfs/InodeTree.java | 51 ++++++++--- .../hadoop/fs/viewfs/ViewFileSystem.java | 85 ++++++++++++++----- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 42 ++++++--- .../hadoop/fs/viewfs/TestViewFsConfig.java | 7 +- .../fs/viewfs/ViewFileSystemBaseTest.java | 44 +++++++++- .../fs/viewfs/TestViewFileSystemHdfs.java | 76 +++++++++++++++++ .../hadoop/fs/viewfs/TestViewFsHdfs.java | 78 +++++++++++++++++ 7 files changed, 338 insertions(+), 45 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 0a949fed827b2..779cec875b65f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; +import com.google.common.base.Function; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -158,8 +159,10 @@ void addLink(final String pathComponent, final INodeLink link) static class INodeLink extends INode { final boolean isMergeLink; // true if MergeLink final URI[] targetDirLinkList; - final T targetFileSystem; // file system object created from the link. - + private T targetFileSystem; // file system object created from the link. + // Function to initialize file system. Only applicable for simple links + private Function fileSystemInitMethod; + private final Object lock = new Object(); /** * Construct a mergeLink */ @@ -175,12 +178,14 @@ static class INodeLink extends INode { * Construct a simple link (i.e. not a mergeLink) */ INodeLink(final String pathToNode, final UserGroupInformation aUgi, - final T targetFs, final URI aTargetDirLink) { + Function createFileSystemMethod, + final URI aTargetDirLink) { super(pathToNode, aUgi); - targetFileSystem = targetFs; + targetFileSystem = null; targetDirLinkList = new URI[1]; targetDirLinkList[0] = aTargetDirLink; isMergeLink = false; + this.fileSystemInitMethod = createFileSystemMethod; } /** @@ -196,6 +201,33 @@ Path getTargetLink() { } return new Path(result.toString()); } + + /** + * Get the instance of FileSystem to use, creating one if needed. + * @return An Initialized instance of T + * @throws IOException + */ + public T getTargetFileSystem() throws IOException { + if (targetFileSystem != null) { + return targetFileSystem; + } + // For non NFLY and MERGE links, we initialize the FileSystem when the + // corresponding mount path is accessed. + if (targetDirLinkList.length == 1) { + synchronized (lock) { + if (targetFileSystem != null) { + return targetFileSystem; + } + targetFileSystem = fileSystemInitMethod.apply(targetDirLinkList[0]); + if (targetFileSystem == null) { + throw new IOException( + "Could not initialize target File System for URI : " + + targetDirLinkList[0]); + } + } + } + return targetFileSystem; + } } @@ -258,7 +290,7 @@ private void createLink(final String src, final String target, getTargetFileSystem(targetsListURI), targetsListURI); } else { newLink = new INodeLink(fullPath, aUgi, - getTargetFileSystem(new URI(target)), new URI(target)); + initAndGetTargetFs(), new URI(target)); } curInode.addLink(iPath, newLink); mountPoints.add(new MountPoint(src, newLink)); @@ -267,14 +299,13 @@ private void createLink(final String src, final String target, /** * Below the "public" methods of InodeTree */ - + /** * The user of this class must subclass and implement the following * 3 abstract methods. * @throws IOException */ - protected abstract T getTargetFileSystem(final URI uri) - throws UnsupportedFileSystemException, URISyntaxException, IOException; + protected abstract Function initAndGetTargetFs(); protected abstract T getTargetFileSystem(final INodeDir dir) throws URISyntaxException; @@ -385,7 +416,7 @@ boolean isInternalDir() { * @throws FileNotFoundException */ ResolveResult resolve(final String p, final boolean resolveLastComponent) - throws FileNotFoundException { + throws IOException { // TO DO: - more efficient to not split the path, but simply compare String[] path = breakIntoPathComponents(p); if (path.length <= 1) { // special case for when path is "/" @@ -422,7 +453,7 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent) } final ResolveResult res = new ResolveResult(ResultKind.isExternalDir, - link.targetFileSystem, nextInode.fullPath, remainingPath); + link.getTargetFileSystem(), nextInode.fullPath, remainingPath); return res; } else if (nextInode instanceof INodeDir) { curInode = (INodeDir) nextInode; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 79600483e9793..5ee706d295b43 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -21,10 +21,12 @@ import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT; +import com.google.common.base.Function; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -237,7 +239,7 @@ public void initialize(final URI theUri, final Configuration conf) config = conf; enableInnerCache = config.getBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT); - final InnerCache innerCache = new InnerCache(); + cache = new InnerCache(); // Now build client side view (i.e. client side mount table) from config. final String authority = theUri.getAuthority(); try { @@ -245,16 +247,32 @@ public void initialize(final URI theUri, final Configuration conf) fsState = new InodeTree(conf, authority) { @Override - protected - FileSystem getTargetFileSystem(final URI uri) - throws URISyntaxException, IOException { - FileSystem fs; - if (enableInnerCache) { - fs = innerCache.get(uri, config); - } else { - fs = FileSystem.get(uri, config); + protected Function initAndGetTargetFs() { + return new Function() { + @Override + public FileSystem apply(final URI uri) { + FileSystem fs; + try { + fs = ugi.doAs(new PrivilegedExceptionAction() { + @Override + public FileSystem run() throws IOException { + if (enableInnerCache) { + synchronized (cache) { + return cache.get(uri, config); + } + } else { + return FileSystem.get(uri, config); + } + } + }); + return new ChRootedFileSystem(fs, uri); + } catch (IOException | InterruptedException ex) { + LOG.error("Could not initialize the underlying FileSystem " + + "object. Exception: " + ex.toString()); + } + return null; } - return new ChRootedFileSystem(fs, uri); + }; } @Override @@ -273,12 +291,6 @@ FileSystem getTargetFileSystem(URI[] mergeFsURIList) } }; - if (enableInnerCache) { - // All fs instances are created and cached on startup. The cache is - // readonly after the initialize() so the concurrent access of the cache - // is safe. - cache = innerCache.unmodifiableCache(); - } workingDir = this.getHomeDirectory(); renameStrategy = RenameStrategy.valueOf( conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, @@ -311,7 +323,7 @@ public ViewFileSystem(final Configuration conf) throws IOException { this(FsConstants.VIEWFS_URI, conf); } - public Path getTrashCanLocation(final Path f) throws FileNotFoundException { + public Path getTrashCanLocation(final Path f) throws IOException { final InodeTree.ResolveResult res = fsState.resolve(getUriPath(f), true); return res.isInternalDir() ? null : res.targetFileSystem.getHomeDirectory(); @@ -767,9 +779,34 @@ public void removeXAttr(Path path, String name) throws IOException { public void setVerifyChecksum(final boolean verifyChecksum) { List> mountPoints = fsState.getMountPoints(); + Map fsMap = initializeMountedFileSystems(mountPoints); + for (InodeTree.MountPoint mount : mountPoints) { + fsMap.get(mount.src).setVerifyChecksum(verifyChecksum); + } + } + + /** + * Initialize the target filesystem for all mount points. + * @param mountPoints The mount points + * @return Mapping of mount point and the initialized target filesystems + * @throws RuntimeException when the target file system cannot be initialized + */ + private Map initializeMountedFileSystems( + List> mountPoints) { + FileSystem fs = null; + Map fsMap = new HashMap<>(mountPoints.size()); for (InodeTree.MountPoint mount : mountPoints) { - mount.target.targetFileSystem.setVerifyChecksum(verifyChecksum); + try { + fs = mount.target.getTargetFileSystem(); + fsMap.put(mount.src, fs); + } catch (IOException ex) { + String errMsg = "Not able to initialize FileSystem for mount path " + + mount.src + " with exception " + ex; + LOG.error(errMsg); + throw new RuntimeException(errMsg, ex); + } } + return fsMap; } @Override @@ -795,6 +832,9 @@ public long getDefaultBlockSize(Path f) { return res.targetFileSystem.getDefaultBlockSize(res.remainingPath); } catch (FileNotFoundException e) { throw new NotInMountpointException(f, "getDefaultBlockSize"); + } catch (IOException e) { + throw new RuntimeException("Not able to initialize fs in " + + " getDefaultBlockSize for path " + f + " with exception", e); } } @@ -806,6 +846,9 @@ public short getDefaultReplication(Path f) { return res.targetFileSystem.getDefaultReplication(res.remainingPath); } catch (FileNotFoundException e) { throw new NotInMountpointException(f, "getDefaultReplication"); + } catch (IOException e) { + throw new RuntimeException("Not able to initialize fs in " + + " getDefaultReplication for path " + f + " with exception", e); } } @@ -834,8 +877,9 @@ public QuotaUsage getQuotaUsage(Path f) throws IOException { public void setWriteChecksum(final boolean writeChecksum) { List> mountPoints = fsState.getMountPoints(); + Map fsMap = initializeMountedFileSystems(mountPoints); for (InodeTree.MountPoint mount : mountPoints) { - mount.target.targetFileSystem.setWriteChecksum(writeChecksum); + fsMap.get(mount.src).setWriteChecksum(writeChecksum); } } @@ -843,9 +887,10 @@ public void setWriteChecksum(final boolean writeChecksum) { public FileSystem[] getChildFileSystems() { List> mountPoints = fsState.getMountPoints(); + Map fsMap = initializeMountedFileSystems(mountPoints); Set children = new HashSet(); for (InodeTree.MountPoint mountPoint : mountPoints) { - FileSystem targetFs = mountPoint.target.targetFileSystem; + FileSystem targetFs = fsMap.get(mountPoint.src); children.addAll(Arrays.asList(targetFs.getChildFileSystems())); } return children.toArray(new FileSystem[]{}); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 364485fe29fed..1440a4b6d5b70 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -19,10 +19,12 @@ import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import com.google.common.base.Function; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -65,6 +67,8 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -152,6 +156,7 @@ @InterfaceAudience.Public @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */ public class ViewFs extends AbstractFileSystem { + static final Logger LOG = LoggerFactory.getLogger(ViewFs.class); final long creationTime; // of the the mount table final UserGroupInformation ugi; // the user/group of user who created mtable final Configuration config; @@ -212,16 +217,32 @@ public ViewFs(final Configuration conf) throws IOException, fsState = new InodeTree(conf, authority) { @Override - protected - AbstractFileSystem getTargetFileSystem(final URI uri) - throws URISyntaxException, UnsupportedFileSystemException { - String pathString = uri.getPath(); - if (pathString.isEmpty()) { - pathString = "/"; + protected Function initAndGetTargetFs() { + return new Function() { + @Override + public AbstractFileSystem apply(final URI uri) { + AbstractFileSystem fs; + try { + fs = ugi.doAs( + new PrivilegedExceptionAction() { + @Override + public AbstractFileSystem run() throws IOException { + return AbstractFileSystem.createFileSystem(uri, config); + } + }); + String pathString = uri.getPath(); + if (pathString.isEmpty()) { + pathString = "/"; + } + return new ChRootedFs(fs, new Path(pathString)); + } catch (IOException | URISyntaxException | + InterruptedException ex) { + LOG.error("Could not initialize underlying FileSystem object" + +" for uri " + uri + "with exception: " + ex.toString()); + } + return null; } - return new ChRootedFs( - AbstractFileSystem.createFileSystem(uri, config), - new Path(pathString)); + }; } @Override @@ -624,7 +645,8 @@ public List> getDelegationTokens(String renewer) throws IOException { List> result = new ArrayList>(initialListSize); for ( int i = 0; i < mountPoints.size(); ++i ) { List> tokens = - mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer); + mountPoints.get(i).target.getTargetFileSystem() + .getDelegationTokens(renewer); if (tokens != null) { result.addAll(tokens); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java index 75b329ca6a35e..daf68c860dc60 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; +import com.google.common.base.Function; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -47,10 +48,8 @@ class Foo { }; new InodeTree(conf, null) { @Override - protected - Foo getTargetFileSystem(final URI uri) - throws URISyntaxException, UnsupportedFileSystemException { - return null; + protected Function initAndGetTargetFs() { + return null; } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 025fe67863566..d24d92a152a24 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -36,6 +36,7 @@ import static org.apache.hadoop.fs.FileSystemTestHelper.*; import org.apache.hadoop.fs.permission.AclEntry; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.test.GenericTestUtils.*; import static org.junit.Assert.*; @@ -65,7 +66,6 @@ import org.junit.Test; import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.*; /** *

@@ -1037,6 +1037,8 @@ public void testChildrenFileSystemLeak() throws Exception { final int cacheSize = TestFileUtil.getCacheSize(); ViewFileSystem viewFs = (ViewFileSystem) FileSystem .get(new URI("viewfs://" + clusterName + "/"), config); + viewFs.resolvePath( + new Path(String.format("viewfs://%s/%s", clusterName, "/user"))); assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); viewFs.close(); assertEquals(cacheSize, TestFileUtil.getCacheSize()); @@ -1087,4 +1089,44 @@ public void testConfLinkSlash() throws Exception { } } + @Test + public void testTargetFileSystemLazyInitialization() throws Exception { + final String clusterName = "cluster" + new Random().nextInt(); + Configuration config = new Configuration(conf); + config.setBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); + config.setClass("fs.mockfs.impl", + TestChRootedFileSystem.MockFileSystem.class, FileSystem.class); + ConfigUtil.addLink(config, clusterName, "/user", + URI.create("mockfs://mockauth1/mockpath")); + ConfigUtil.addLink(config, clusterName, + "/mock", URI.create("mockfs://mockauth/mockpath")); + + final int cacheSize = TestFileUtil.getCacheSize(); + ViewFileSystem viewFs = (ViewFileSystem) FileSystem + .get(new URI("viewfs://" + clusterName + "/"), config); + + // As no inner file system instance has been initialized, + // cache size will remain the same + // cache is disabled for viewfs scheme, so the viewfs:// instance won't + // go in the cache even after the initialization + assertEquals(cacheSize, TestFileUtil.getCacheSize()); + + // This resolve path will initialize the file system corresponding + // to the mount table entry of the path "/user" + viewFs.resolvePath( + new Path(String.format("viewfs://%s/%s", clusterName, "/user"))); + + // Cache size will increase by 1. + assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); + // This resolve path will initialize the file system corresponding + // to the mount table entry of the path "/mock" + viewFs.resolvePath(new Path(String.format("viewfs://%s/%s", clusterName, + "/mock"))); + // One more file system instance will get initialized. + assertEquals(cacheSize + 2, TestFileUtil.getCacheSize()); + viewFs.close(); + // Initialized FileSystem instances will not be removed from cache as + // viewfs inner cache is disabled + assertEquals(cacheSize + 2, TestFileUtil.getCacheSize()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 7a31a37dcce57..b8da0c54903cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -20,7 +20,10 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.Map; import javax.security.auth.login.LoginException; import org.apache.hadoop.conf.Configuration; @@ -30,12 +33,17 @@ import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; + +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.junit.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { @@ -155,4 +163,72 @@ public void testRenameAccorssFilesystem() throws IOException { e); } } + + @Test + public void testTargetFileSystemLazyInitializationWithUgi() throws Exception { + final Map map = new HashMap<>(); + final Path user1Path = new Path("/data/user1"); + + // Scenario - 1: Create FileSystem with the current user context + // Both mkdir and delete should be successful + FileSystem fs = FileSystem.get(FsConstants.VIEWFS_URI, conf); + fs.mkdirs(user1Path); + fs.delete(user1Path, false); + + // Scenario - 2: Create FileSystem with the a different user context + final UserGroupInformation userUgi = UserGroupInformation + .createUserForTesting("user1@HADOOP.COM", new String[]{"hadoop"}); + userUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + String doAsUserName = ugi.getUserName(); + assertEquals(doAsUserName, "user1@HADOOP.COM"); + + FileSystem viewFS = FileSystem.get(FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Try creating a directory with the file context created by a different ugi + // Though we are running the mkdir with the current user context, the + // target filesystem will be instantiated by the ugi with which the + // file context was created. + try { + FileSystem otherfs = map.get("user1"); + otherfs.mkdirs(user1Path); + fail("This mkdir should fail"); + } catch (AccessControlException ex) { + // Exception is expected as the FileSystem was created with ugi of user1 + // So when we are trying to access the /user/user1 path for the first + // time, the corresponding file system is initialized and it tries to + // execute the task with ugi with which the FileSystem was created. + } + + // Change the permission of /data path so that user1 can create a directory + fsTarget.setOwner(new Path(targetTestRoot, "data"), + "user1", "test2"); + // set permission of target to allow rename to target + fsTarget.setPermission(new Path(targetTestRoot, "data"), + new FsPermission("775")); + + userUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws IOException { + FileSystem viewFS = FileSystem.get(FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Although we are running with current user context, and current user + // context does not have write permission, we are able to create the + // directory as its using ugi of user1 which has write permission. + FileSystem otherfs = map.get("user1"); + otherfs.mkdirs(user1Path); + String owner = otherfs.getFileStatus(user1Path).getOwner(); + assertEquals("The owner did not match ", owner, userUgi.getShortUserName()); + otherfs.delete(user1Path, false); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java index fda667251b8e0..540883dd2e208 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java @@ -21,18 +21,28 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.Map; import javax.security.auth.login.LoginException; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class TestViewFsHdfs extends ViewFsBaseTest { @@ -85,5 +95,73 @@ public void setUp() throws Exception { int getExpectedDelegationTokenCount() { return 8; } + + @Test + public void testTargetFileSystemLazyInitialization() throws Exception { + final Map map = new HashMap<>(); + final Path user1Path = new Path("/data/user1"); + + // Scenario - 1: Create FileContext with the current user context + // Both mkdir and delete should be successful + FileContext fs = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf); + fs.mkdir(user1Path, FileContext.DEFAULT_PERM, false); + fs.delete(user1Path, false); + + // Scenario - 2: Create FileContext with the a different user context + final UserGroupInformation userUgi = UserGroupInformation + .createUserForTesting("user1@HADOOP.COM", new String[]{"hadoop"}); + userUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + String doAsUserName = ugi.getUserName(); + assertEquals(doAsUserName, "user1@HADOOP.COM"); + + FileContext viewFS = FileContext.getFileContext( + FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Try creating a directory with the file context created by a different ugi + // Though we are running the mkdir with the current user context, the + // target filesystem will be instantiated by the ugi with which the + // file context was created. + try { + FileContext otherfs = map.get("user1"); + otherfs.mkdir(user1Path, FileContext.DEFAULT_PERM, false); + fail("This mkdir should fail"); + } catch (AccessControlException ex) { + // Exception is expected as the FileContext was created with ugi of user1 + // So when we are trying to access the /user/user1 path for the first + // time, the corresponding file system is initialized and it tries to + // execute the task with ugi with which the FileContext was created. + } + + // Change the permission of /data path so that user1 can create a directory + fcView.setOwner(new Path("/data"), "user1", "test2"); + // set permission of target to allow rename to target + fcView.setPermission(new Path("/data"), new FsPermission("775")); + + userUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws IOException { + FileContext viewFS = FileContext.getFileContext( + FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Although we are running with current user context, and current user + // context does not have write permission, we are able to create the + // directory as its using ugi of user1 which has write permission. + FileContext otherfs = map.get("user1"); + otherfs.mkdir(user1Path, FileContext.DEFAULT_PERM, false); + String owner = otherfs.getFileStatus(user1Path).getOwner(); + assertEquals("The owner did not match ", owner, userUgi.getShortUserName()); + otherfs.delete(user1Path, false); + } }