diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 5360d55e10644..466057feec718 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -413,6 +413,11 @@ public T getTargetFileSystem() throws IOException { } return targetFileSystem; } + + T getTargetFileSystemForClose() throws IOException { + return targetFileSystem; + } + } private void createLink(final String src, final String target, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index e31a701a6eaa7..b4cf96ea5996a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1926,6 +1926,26 @@ enum RenameStrategy { SAME_FILESYSTEM_ACROSS_MOUNTPOINT } + private void closeChildFileSystems(FileSystem fs) { + if (fs != null) { + FileSystem[] childFs = fs.getChildFileSystems(); + for (FileSystem child : childFs) { + if (child != null) { + String disableCacheName = String.format("fs.%s.impl.disable.cache", + child.getUri().getScheme()); + if (config.getBoolean(disableCacheName, false)) { + try { + child.close(); + } catch (IOException e) { + LOG.info("Fail closing ViewFileSystem's child filesystem " + fs, + e); + } + } + } + } + } + } + @Override public void close() throws IOException { super.close(); @@ -1933,5 +1953,19 @@ public void close() throws IOException { cache.closeAll(); cache.clear(); } + + if (!enableInnerCache) { + for (InodeTree.MountPoint mountPoint : + fsState.getMountPoints()) { + FileSystem targetFs = mountPoint.target.getTargetFileSystemForClose(); + closeChildFileSystems(targetFs); + } + + if (fsState.isRootInternalDir() && + fsState.getRootFallbackLink() != null) { + closeChildFileSystems( + fsState.getRootFallbackLink().getTargetFileSystem()); + } + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemClose.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemClose.java new file mode 100644 index 0000000000000..7301e0c0acfa3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemClose.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.viewfs; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsConstants; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.AbstractHadoopTestBase; +import org.junit.Test; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +public class TestViewFileSystemClose extends AbstractHadoopTestBase { + + /** + * Verify that all child file systems of a ViewFileSystem will be shut down + * when the cache is disabled. + * @throws IOException + */ + @Test + public void testFileSystemLeak() throws Exception { + + Configuration conf = new Configuration(); + conf.set("fs.viewfs.impl", ViewFileSystem.class.getName()); + conf.setBoolean("fs.viewfs.enable.inner.cache", false); + conf.setBoolean("fs.viewfs.impl.disable.cache", true); + conf.setBoolean("fs.hdfs.impl.disable.cache", true); + + String rootPath = "hdfs://localhost/tmp"; + ConfigUtil.addLink(conf, "/data", new Path(rootPath, "data").toUri()); + ViewFileSystem viewFs = + (ViewFileSystem) FileSystem.get(FsConstants.VIEWFS_URI, conf); + + FileSystem[] children = viewFs.getChildFileSystems(); + viewFs.close(); + FileSystem.closeAll(); + for (FileSystem fs : children) { + intercept(IOException.class, "Filesystem closed", + "Expect Filesystem closed IOException", + () -> fs.create(new Path(rootPath, "neverSuccess"))); + } + } +}