From d9441f95c362214e249b969c9ccc3fb4e8c1709a Mon Sep 17 00:00:00 2001 From: Shashikant Banerjee Date: Tue, 21 Jul 2020 11:13:05 +0530 Subject: [PATCH 001/335] HDFS-15470. Added more unit tests to validate rename behaviour across snapshots. Contributed by Shashikant Banerjee. --- .../namenode/TestFSImageWithSnapshot.java | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java index f27c8f28bd448..3fd725b7f6965 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java @@ -649,4 +649,182 @@ public void testFSImageWithDoubleRename() throws Exception { fsn = cluster.getNamesystem(); hdfs = cluster.getFileSystem(); } + + + @Test (timeout=60000) + public void testFSImageWithRename1() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path rennamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s2"); + Path diry = new Path("/dir1/dira/dirb/diry"); + hdfs.mkdirs(diry); + hdfs.createSnapshot(dir1, "s3"); + Path file1 = new Path("/dir1/dira/dirb/diry/file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, seed); + hdfs.createSnapshot(dir1, "s4"); + hdfs.delete(new Path("/dir1/dira/dirb"), true); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + // file1 should exist in the last snapshot + assertTrue(hdfs.exists( + new Path("/dir1/.snapshot/s4/dira/dirb/diry/file1"))); + + // save namespace and restart cluster + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test (timeout=60000) + public void testFSImageWithRename2() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path rennamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s2"); + Path file1 = new Path("/dir1/dira/dirb/file1"); + DFSTestUtil.createFile(hdfs, + new Path( + "/dir1/dira/dirb/file1"), BLOCKSIZE, (short) 1, seed); + hdfs.createSnapshot(dir1, "s3"); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + assertTrue(hdfs.exists(file1)); + + // save namespace and restart cluster + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test(timeout = 60000) + public void testFSImageWithRename3() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path rennamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, rennamePath); + hdfs.createSnapshot(dir1, "s2"); + Path diry = new Path("/dir1/dira/dirb/diry"); + hdfs.mkdirs(diry); + hdfs.createSnapshot(dir1, "s3"); + Path file1 = new Path("/dir1/dira/dirb/diry/file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, seed); + hdfs.createSnapshot(dir1, "s4"); + hdfs.delete(new Path("/dir1/dira/dirb"), true); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + // file1 should exist in the last snapshot + assertTrue(hdfs.exists(new Path( + "/dir1/.snapshot/s4/dira/dirb/diry/file1"))); + + // save namespace and restart cluster + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + + @Test (timeout=60000) + public void testFSImageWithRename4() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + Path dira = new Path(dir1, "dira"); + Path dirx = new Path(dir1, "dirx"); + Path dirb = new Path(dirx, "dirb"); + hdfs.mkdirs(dira); + hdfs.mkdirs(dirx); + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + hdfs.mkdirs(dirb); + hdfs.createSnapshot(dir1, "s1"); + Path renamePath = new Path(dira, "dirb"); + // mv /dir1/dirx/dirb to /dir1/dira/dirb + hdfs.rename(dirb, renamePath); + hdfs.createSnapshot(dir1, "s2"); + Path diry = new Path("/dir1/dira/dirb/diry"); + hdfs.mkdirs(diry); + hdfs.createSnapshot(dir1, "s3"); + Path file1 = new Path("/dir1/dira/dirb/diry/file1"); + DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, (short) 1, seed); + hdfs.createSnapshot(dir1, "s4"); + hdfs.delete(new Path("/dir1/dira/dirb/diry/file1"), false); + hdfs.deleteSnapshot(dir1, "s1"); + hdfs.deleteSnapshot(dir1, "s3"); + // file1 should exist in the last snapshot + assertTrue(hdfs.exists( + new Path("/dir1/.snapshot/s4/dira/dirb/diry/file1"))); + + // save namespace and restart cluster + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.shutdown(); + cluster = new MiniDFSCluster.Builder(conf).format(false) + .numDataNodes(NUM_DATANODES).build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + } + } From d57462f2daee5f057e32219d4123a3f75506d6d4 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Mon, 20 Jul 2020 23:06:24 -0700 Subject: [PATCH 002/335] HDFS-15479. Ordered snapshot deletion: make it a configurable feature (#2156) --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 7 +- .../hdfs/server/namenode/FSDirSnapshotOp.java | 37 +++++- .../hdfs/server/namenode/FSDirectory.java | 20 ++++ .../DirectorySnapshottableFeature.java | 2 +- .../namenode/TestOrderedSnapshotDeletion.java | 105 ++++++++++++++++++ 5 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 9de33ff60a62e..dd24785688270 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -500,8 +500,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_SNAPSHOT_MAX_LIMIT = "dfs.namenode.snapshot.max.limit"; - public static final int DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT = 65536; + + public static final String DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED = + "dfs.namenode.snapshot.deletion.ordered"; + public static final boolean DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT + = false; + public static final String DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL = "dfs.namenode.snapshot.skiplist.interval"; public static final int DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL_DEFAULT = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java index c854f83142e5f..c2eb401c5fbb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.FSLimitException; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -249,12 +250,41 @@ static INode.BlocksMapUpdateInfo deleteSnapshot( fsd.checkOwner(pc, iip); } + // time of snapshot deletion + final long now = Time.now(); + if (fsd.isSnapshotDeletionOrdered()) { + final INodeDirectory srcRoot = snapshotManager.getSnapshottableRoot(iip); + final DirectorySnapshottableFeature snapshottable + = srcRoot.getDirectorySnapshottableFeature(); + final Snapshot snapshot = snapshottable.getSnapshotByName( + srcRoot, snapshotName); + + // Diffs must be not empty since a snapshot exists in the list + final int earliest = snapshottable.getDiffs().iterator().next() + .getSnapshotId(); + if (snapshot.getId() != earliest) { + throw new SnapshotException("Failed to delete snapshot " + snapshotName + + " from directory " + srcRoot.getFullPathName() + + ": " + snapshot + " is not the earliest snapshot id=" + earliest + + " (" + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + + " is " + fsd.isSnapshotDeletionOrdered() + ")"); + } + } + + final INode.BlocksMapUpdateInfo collectedBlocks = deleteSnapshot( + fsd, snapshotManager, iip, snapshotName, now); + fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName, + logRetryCache, now); + return collectedBlocks; + } + + static INode.BlocksMapUpdateInfo deleteSnapshot( + FSDirectory fsd, SnapshotManager snapshotManager, INodesInPath iip, + String snapshotName, long now) throws IOException { INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo(); ChunkedArrayList removedINodes = new ChunkedArrayList<>(); INode.ReclaimContext context = new INode.ReclaimContext( fsd.getBlockStoragePolicySuite(), collectedBlocks, removedINodes, null); - // time of snapshot deletion - final long now = Time.now(); fsd.writeLock(); try { snapshotManager.deleteSnapshot(iip, snapshotName, context, now); @@ -266,9 +296,6 @@ static INode.BlocksMapUpdateInfo deleteSnapshot( fsd.writeUnlock(); } removedINodes.clear(); - fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName, - logRetryCache, now); - return collectedBlocks; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index fb5c9df8debdf..6df255e86574e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -187,6 +187,7 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) { private boolean posixAclInheritanceEnabled; private final boolean xattrsEnabled; private final int xattrMaxSize; + private final boolean snapshotDeletionOrdered; // precision of access times. private final long accessTimePrecision; @@ -353,6 +354,20 @@ public enum DirOp { + " hard limit " + DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_HARD_LIMIT + ": (%s).", DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY); + this.snapshotDeletionOrdered = + conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT); + LOG.info("{} = {}", DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, + snapshotDeletionOrdered); + if (snapshotDeletionOrdered && !xattrsEnabled) { + throw new HadoopIllegalArgumentException("" + + "XAttrs is required by snapshotDeletionOrdered:" + + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + + " is true but " + + DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY + + " is false."); + } + this.accessTimePrecision = conf.getLong( DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT); @@ -610,6 +625,11 @@ boolean isXattrsEnabled() { return xattrsEnabled; } int getXattrMaxSize() { return xattrMaxSize; } + + boolean isSnapshotDeletionOrdered() { + return snapshotDeletionOrdered; + } + boolean isAccessTimeSupported() { return accessTimePrecision > 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index b38d8bfe8ce06..fe97cf87015e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -358,7 +358,7 @@ SnapshotDiffListingInfo computeDiff(final INodeDirectory snapshotRootDir, * @throws SnapshotException If snapshotName is not null or empty, but there * is no snapshot matching the name. */ - private Snapshot getSnapshotByName(INodeDirectory snapshotRoot, + public Snapshot getSnapshotByName(INodeDirectory snapshotRoot, String snapshotName) throws SnapshotException { Snapshot s = null; if (snapshotName != null && !snapshotName.isEmpty()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java new file mode 100644 index 0000000000000..c8df780465bab --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.SnapshotException; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; + +import java.io.IOException; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; + +/** Test ordered snapshot deletion. */ +public class TestOrderedSnapshotDeletion { + static final Logger LOG = LoggerFactory.getLogger(FSDirectory.class); + + { + SnapshotTestHelper.disableLogs(); + GenericTestUtils.setLogLevel(INode.LOG, Level.TRACE); + } + + private final Path snapshottableDir + = new Path("/" + getClass().getSimpleName()); + + private MiniDFSCluster cluster; + + @Before + public void setUp() throws Exception { + final Configuration conf = new Configuration(); + conf.setBoolean(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, true); + + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster.waitActive(); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Test (timeout=60000) + public void testConf() throws Exception { + DistributedFileSystem hdfs = cluster.getFileSystem(); + hdfs.mkdirs(snapshottableDir); + hdfs.allowSnapshot(snapshottableDir); + + final Path sub0 = new Path(snapshottableDir, "sub0"); + hdfs.mkdirs(sub0); + hdfs.createSnapshot(snapshottableDir, "s0"); + + final Path sub1 = new Path(snapshottableDir, "sub1"); + hdfs.mkdirs(sub1); + hdfs.createSnapshot(snapshottableDir, "s1"); + + final Path sub2 = new Path(snapshottableDir, "sub2"); + hdfs.mkdirs(sub2); + hdfs.createSnapshot(snapshottableDir, "s2"); + + assertDeletionDenied(snapshottableDir, "s1", hdfs); + assertDeletionDenied(snapshottableDir, "s2", hdfs); + hdfs.deleteSnapshot(snapshottableDir, "s0"); + assertDeletionDenied(snapshottableDir, "s2", hdfs); + hdfs.deleteSnapshot(snapshottableDir, "s1"); + hdfs.deleteSnapshot(snapshottableDir, "s2"); + } + + static void assertDeletionDenied(Path snapshottableDir, String snapshot, + DistributedFileSystem hdfs) throws IOException { + try { + hdfs.deleteSnapshot(snapshottableDir, snapshot); + Assert.fail("deleting " +snapshot + " should fail"); + } catch (SnapshotException se) { + LOG.info("Good, it is expected to have " + se); + } + } +} From 8b7695bb2628574b4450bac19c12b29db9ee0628 Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Tue, 21 Jul 2020 09:00:07 -0700 Subject: [PATCH 003/335] HDFS-15246. ArrayIndexOfboundsException in BlockManager CreateLocatedBlock. Contributed by Hemanth Boyina. --- .../hdfs/server/namenode/FSDirRenameOp.java | 7 +++- .../server/namenode/TestFileTruncate.java | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java index 602f9962942be..a0c1e3a8814e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java @@ -650,7 +650,12 @@ private static class RenameOperation { // snapshot is taken on the dst tree, changes will be recorded in the // latest snapshot of the src tree. if (isSrcInSnapshot) { - srcChild.recordModification(srcLatestSnapshotId); + if (srcChild.isFile()) { + INodeFile file = srcChild.asFile(); + file.recordModification(srcLatestSnapshotId, true); + } else { + srcChild.recordModification(srcLatestSnapshotId); + } } // check srcChild for reference diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java index 7bfa8aedc063c..eef1dba3fa447 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java @@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -67,6 +68,7 @@ import org.apache.hadoop.util.ToolRunner; import org.slf4j.event.Level; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -1393,4 +1395,37 @@ public void testQuotaSpaceConsumedWithSnapshots() throws IOException { fs.getQuotaUsage(root).getSpaceConsumed()); } + + /** + * Test truncate on a snapshotted file. + */ + @Test + public void testTruncatewithRenameandSnapshot() throws Exception { + final Path dir = new Path("/dir"); + fs.mkdirs(dir, new FsPermission((short) 0777)); + final Path file = new Path(dir, "file"); + final Path movedFile = new Path("/file"); + + // 1. create a file and snapshot for dir which is having a file + DFSTestUtil.createFile(fs, file, 10, (short) 3, 0); + fs.allowSnapshot(dir); + Path snapshotPath = fs.createSnapshot(dir, "s0"); + assertTrue(fs.exists(snapshotPath)); + + // 2. move the file + fs.rename(file, new Path("/")); + + // 3.truncate the moved file + final boolean isReady = fs.truncate(movedFile, 5); + if (!isReady) { + checkBlockRecovery(movedFile); + } + FileStatus fileStatus = fs.getFileStatus(movedFile); + assertEquals(5, fileStatus.getLen()); + + // 4. get block locations of file which is in snapshot + LocatedBlocks locations = fs.getClient().getNamenode() + .getBlockLocations("/dir/.snapshot/s0/file", 0, 10); + assertEquals(10, locations.get(0).getBlockSize()); + } } From b4b23ef0d1a0afe6251370a61f922ecdb1624165 Mon Sep 17 00:00:00 2001 From: bilaharith <52483117+bilaharith@users.noreply.github.com> Date: Tue, 21 Jul 2020 21:48:54 +0530 Subject: [PATCH 004/335] HADOOP-17092. ABFS: Making AzureADAuthenticator.getToken() throw HttpException - Contributed by Bilahari T H --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 27 ++++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 5 + .../azurebfs/constants/ConfigurationKeys.java | 6 + .../constants/FileSystemConfigurations.java | 8 ++ .../azurebfs/oauth2/AzureADAuthenticator.java | 42 +++++- .../services/ExponentialRetryPolicy.java | 23 ++++ .../hadoop-azure/src/site/markdown/abfs.md | 20 ++- .../services/TestAzureADAuthenticator.java | 120 ++++++++++++++++++ 8 files changed, 244 insertions(+), 7 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAzureADAuthenticator.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 4cdb3c95eccfb..85bd37a7702c0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -57,6 +57,7 @@ import org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy; import org.apache.hadoop.fs.azurebfs.services.KeyProvider; import org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; @@ -119,6 +120,26 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT) private int customTokenFetchRetryCount; + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS) + private int oauthTokenFetchRetryCount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL) + private int oauthTokenFetchRetryMinBackoff; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL) + private int oauthTokenFetchRetryMaxBackoff; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF) + private int oauthTokenFetchRetryDeltaBackoff; + @LongConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_SIZE_PROPERTY_NAME, MinValue = 0, MaxValue = MAX_AZURE_BLOCK_SIZE, @@ -795,6 +816,12 @@ boolean validateBoolean(Field field) throws IllegalAccessException, InvalidConfi validator.ThrowIfInvalid()).validate(value); } + public ExponentialRetryPolicy getOauthTokenFetchRetryPolicy() { + return new ExponentialRetryPolicy(oauthTokenFetchRetryCount, + oauthTokenFetchRetryMinBackoff, oauthTokenFetchRetryMaxBackoff, + oauthTokenFetchRetryDeltaBackoff); + } + @VisibleForTesting void setReadBufferSize(int bufferSize) { this.readBufferSize = bufferSize; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 66130d384f5bb..59c2e263b25b9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -79,6 +79,7 @@ import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider; import org.apache.hadoop.fs.azurebfs.extensions.ExtensionHelper; import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; +import org.apache.hadoop.fs.azurebfs.oauth2.AzureADAuthenticator; import org.apache.hadoop.fs.azurebfs.oauth2.IdentityTransformer; import org.apache.hadoop.fs.azurebfs.oauth2.IdentityTransformerInterface; import org.apache.hadoop.fs.azurebfs.services.AbfsAclHelper; @@ -1234,6 +1235,10 @@ private void initializeClient(URI uri, String fileSystemName, AccessTokenProvider tokenProvider = null; SASTokenProvider sasTokenProvider = null; + if (authType == AuthType.OAuth) { + AzureADAuthenticator.init(abfsConfiguration); + } + if (authType == AuthType.SharedKey) { LOG.trace("Fetching SharedKey credentials"); int dotIndex = accountName.indexOf(AbfsHttpConstants.DOT); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index b5feee64ab476..5f1ad31e4876c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -45,6 +45,12 @@ public final class ConfigurationKeys { public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries"; public static final String AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT = "fs.azure.custom.token.fetch.retry.count"; + // Retry strategy for getToken calls + public static final String AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT = "fs.azure.oauth.token.fetch.retry.max.retries"; + public static final String AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF = "fs.azure.oauth.token.fetch.retry.min.backoff.interval"; + public static final String AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF = "fs.azure.oauth.token.fetch.retry.max.backoff.interval"; + public static final String AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF = "fs.azure.oauth.token.fetch.retry.delta.backoff"; + // Read and write buffer sizes defined by the user public static final String AZURE_WRITE_BUFFER_SIZE = "fs.azure.write.request.size"; public static final String AZURE_READ_BUFFER_SIZE = "fs.azure.read.request.size"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index a367daf6ee564..260c496434cac 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -35,6 +35,8 @@ public final class FileSystemConfigurations { public static final String USER_HOME_DIRECTORY_PREFIX = "/user"; + private static final int SIXTY_SECONDS = 60 * 1000; + // Retry parameter defaults. public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s @@ -42,6 +44,12 @@ public final class FileSystemConfigurations { public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; + // Retry parameter defaults. + public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS = 5; + public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL = 0; + public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL = SIXTY_SECONDS; + public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF = 2; + private static final int ONE_KB = 1024; private static final int ONE_MB = ONE_KB * ONE_KB; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java index 435335fc2012d..dfb24a7fdb111 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java @@ -18,9 +18,11 @@ package org.apache.hadoop.fs.azurebfs.oauth2; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Date; @@ -34,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.azurebfs.services.AbfsIoUtils; @@ -56,10 +59,16 @@ public final class AzureADAuthenticator { private static final int CONNECT_TIMEOUT = 30 * 1000; private static final int READ_TIMEOUT = 30 * 1000; + private static ExponentialRetryPolicy tokenFetchRetryPolicy; + private AzureADAuthenticator() { // no operation } + public static void init(AbfsConfiguration abfsConfiguration) { + tokenFetchRetryPolicy = abfsConfiguration.getOauthTokenFetchRetryPolicy(); + } + /** * gets Azure Active Directory token using the user ID and password of * a service principal (that is, Web App in Azure Active Directory). @@ -81,8 +90,7 @@ private AzureADAuthenticator() { * @throws IOException throws IOException if there is a failure in connecting to Azure AD */ public static AzureADToken getTokenUsingClientCreds(String authEndpoint, - String clientId, String clientSecret) - throws IOException { + String clientId, String clientSecret) throws IOException { Preconditions.checkNotNull(authEndpoint, "authEndpoint"); Preconditions.checkNotNull(clientId, "clientId"); Preconditions.checkNotNull(clientSecret, "clientSecret"); @@ -283,13 +291,14 @@ private static AzureADToken getTokenCall(String authEndpoint, String body, Hashtable headers, String httpMethod, boolean isMsi) throws IOException { AzureADToken token = null; - ExponentialRetryPolicy retryPolicy - = new ExponentialRetryPolicy(3, 0, 1000, 2); int httperror = 0; IOException ex = null; boolean succeeded = false; + boolean isRecoverableFailure = true; int retryCount = 0; + boolean shouldRetry; + LOG.trace("First execution of REST operation getTokenSingleCall"); do { httperror = 0; ex = null; @@ -299,17 +308,38 @@ private static AzureADToken getTokenCall(String authEndpoint, String body, httperror = e.httpErrorCode; ex = e; } catch (IOException e) { - ex = e; + httperror = -1; + isRecoverableFailure = isRecoverableFailure(e); + ex = new HttpException(httperror, "", String + .format("AzureADAuthenticator.getTokenCall threw %s : %s", + e.getClass().getTypeName(), e.getMessage()), authEndpoint, "", + ""); } succeeded = ((httperror == 0) && (ex == null)); + shouldRetry = !succeeded && isRecoverableFailure + && tokenFetchRetryPolicy.shouldRetry(retryCount, httperror); retryCount++; - } while (!succeeded && retryPolicy.shouldRetry(retryCount, httperror)); + if (shouldRetry) { + LOG.debug("Retrying getTokenSingleCall. RetryCount = {}", retryCount); + try { + Thread.sleep(tokenFetchRetryPolicy.getRetryInterval(retryCount)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + } while (shouldRetry); if (!succeeded) { throw ex; } return token; } + private static boolean isRecoverableFailure(IOException e) { + return !(e instanceof MalformedURLException + || e instanceof FileNotFoundException); + } + private static AzureADToken getTokenSingleCall(String authEndpoint, String payload, Hashtable headers, String httpMethod, boolean isMsi) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java index b272cf27ca0d7..4cf2f710b6a60 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java @@ -21,6 +21,8 @@ import java.util.Random; import java.net.HttpURLConnection; +import com.google.common.annotations.VisibleForTesting; + /** * Retry policy used by AbfsClient. * */ @@ -138,4 +140,25 @@ public long getRetryInterval(final int retryCount) { return retryInterval; } + + @VisibleForTesting + int getRetryCount() { + return this.retryCount; + } + + @VisibleForTesting + int getMinBackoff() { + return this.minBackoff; + } + + @VisibleForTesting + int getMaxBackoff() { + return maxBackoff; + } + + @VisibleForTesting + int getDeltaBackoff() { + return this.deltaBackoff; + } + } diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index a8ac08373d46c..4640bab2c12dc 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -330,6 +330,22 @@ All secrets can be stored in JCEKS files. These are encrypted and password protected —use them or a compatible Hadoop Key Management Store wherever possible +### AAD Token fetch retries + +The exponential retry policy used for the AAD token fetch retries can be tuned +with the following configurations. +* `fs.azure.oauth.token.fetch.retry.max.retries`: Sets the maximum number of + retries. Default value is 5. +* `fs.azure.oauth.token.fetch.retry.min.backoff.interval`: Minimum back-off + interval. Added to the retry interval computed from delta backoff. By + default this si set as 0. Set the interval in milli seconds. +* `fs.azure.oauth.token.fetch.retry.max.backoff.interval`: Maximum back-off +interval. Default value is 60000 (sixty seconds). Set the interval in milli +seconds. +* `fs.azure.oauth.token.fetch.retry.delta.backoff`: Back-off interval between +retries. Multiples of this timespan are used for subsequent retry attempts + . The default value is 2. + ### Default: Shared Key This is the simplest authentication mechanism of account + password. @@ -776,7 +792,9 @@ bytes. The value should be between 16384 to 104857600 both inclusive (16 KB to `fs.azure.readaheadqueue.depth`: Sets the readahead queue depth in AbfsInputStream. In case the set value is negative the read ahead queue depth will be set as Runtime.getRuntime().availableProcessors(). By default the value -will be -1. +will be -1. To disable readaheads, set this value to 0. If your workload is + doing only random reads (non-sequential) or you are seeing throttling, you + may try setting this value to 0. ### Security Options `fs.azure.always.use.https`: Enforces to use HTTPS instead of HTTP when the flag diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAzureADAuthenticator.java new file mode 100644 index 0000000000000..8e79288cf6e7d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAzureADAuthenticator.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_NAME; + +public class TestAzureADAuthenticator extends AbstractAbfsIntegrationTest { + + private static final int TEST_RETRY_COUNT = 10; + private static final int TEST_MIN_BACKOFF = 20; + private static final int TEST_MAX_BACKOFF = 30; + private static final int TEST_DELTA_BACKOFF = 40; + + public TestAzureADAuthenticator() throws Exception { + super(); + } + + @Test + public void testDefaultOAuthTokenFetchRetryPolicy() throws Exception { + getConfiguration().unset(AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT); + getConfiguration().unset(AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF); + getConfiguration().unset(AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF); + getConfiguration().unset(AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF); + + String accountName = getConfiguration().get(FS_AZURE_ACCOUNT_NAME); + AbfsConfiguration abfsConfig = new AbfsConfiguration(getRawConfiguration(), + accountName); + + ExponentialRetryPolicy retryPolicy = abfsConfig + .getOauthTokenFetchRetryPolicy(); + + Assertions.assertThat(retryPolicy.getRetryCount()).describedAs( + "retryCount should be the default value {} as the same " + + "is not configured", + DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS) + .isEqualTo(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS); + Assertions.assertThat(retryPolicy.getMinBackoff()).describedAs( + "minBackOff should be the default value {} as the same is " + + "not configured", + DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL) + .isEqualTo(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL); + Assertions.assertThat(retryPolicy.getMaxBackoff()).describedAs( + "maxBackOff should be the default value {} as the same is " + + "not configured", + DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL) + .isEqualTo(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL); + Assertions.assertThat(retryPolicy.getDeltaBackoff()).describedAs( + "deltaBackOff should be the default value {} as the same " + "is " + + "not configured", + DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF) + .isEqualTo(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF); + + } + + @Test + public void testOAuthTokenFetchRetryPolicy() + throws IOException, IllegalAccessException { + + getConfiguration() + .set(AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT, String.valueOf(TEST_RETRY_COUNT)); + getConfiguration().set(AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF, + String.valueOf(TEST_MIN_BACKOFF)); + getConfiguration().set(AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF, + String.valueOf(TEST_MAX_BACKOFF)); + getConfiguration().set(AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF, + String.valueOf(TEST_DELTA_BACKOFF)); + + String accountName = getConfiguration().get(FS_AZURE_ACCOUNT_NAME); + AbfsConfiguration abfsConfig = new AbfsConfiguration(getRawConfiguration(), + accountName); + + ExponentialRetryPolicy retryPolicy = abfsConfig + .getOauthTokenFetchRetryPolicy(); + + Assertions.assertThat(retryPolicy.getRetryCount()) + .describedAs("retryCount should be {}", TEST_RETRY_COUNT) + .isEqualTo(TEST_RETRY_COUNT); + Assertions.assertThat(retryPolicy.getMinBackoff()) + .describedAs("minBackOff should be {}", TEST_MIN_BACKOFF) + .isEqualTo(TEST_MIN_BACKOFF); + Assertions.assertThat(retryPolicy.getMaxBackoff()) + .describedAs("maxBackOff should be {}", TEST_MAX_BACKOFF) + .isEqualTo(TEST_MAX_BACKOFF); + Assertions.assertThat(retryPolicy.getDeltaBackoff()) + .describedAs("deltaBackOff should be {}", TEST_DELTA_BACKOFF) + .isEqualTo(TEST_DELTA_BACKOFF); + } + +} From d23cc9d85d887f01d72180bdf1af87dfdee15c5a Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Tue, 21 Jul 2020 21:52:38 +0530 Subject: [PATCH 005/335] Hadoop 17132. ABFS: Fix Rename and Delete Idempotency check trigger - Contributed by Sneha Vijayarajan --- .../fs/azurebfs/services/AbfsClient.java | 55 ++++++++++---- .../azurebfs/services/AbfsRestOperation.java | 7 +- .../ITestAzureBlobFileSystemDelete.java | 65 +++++++++++++++- .../ITestAzureBlobFileSystemRename.java | 68 +++++++++++++++++ .../fs/azurebfs/services/TestAbfsClient.java | 76 +++++++++++++++++++ 5 files changed, 253 insertions(+), 18 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index f747bd068ccae..e1ea75e4756de 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -336,10 +336,19 @@ public AbfsRestOperation renamePath(String source, final String destination, fin url, requestHeaders); Instant renameRequestStartTime = Instant.now(); - op.execute(); - - if (op.getResult().getStatusCode() != HttpURLConnection.HTTP_OK) { - return renameIdempotencyCheckOp(renameRequestStartTime, op, destination); + try { + op.execute(); + } catch (AzureBlobFileSystemException e) { + final AbfsRestOperation idempotencyOp = renameIdempotencyCheckOp( + renameRequestStartTime, op, destination); + if (idempotencyOp.getResult().getStatusCode() + == op.getResult().getStatusCode()) { + // idempotency did not return different result + // throw back the exception + throw e; + } else { + return idempotencyOp; + } } return op; @@ -369,14 +378,21 @@ public AbfsRestOperation renameIdempotencyCheckOp( // exists. Check on destination status and if it has a recent LMT timestamp. // If yes, return success, else fall back to original rename request failure response. - final AbfsRestOperation destStatusOp = getPathStatus(destination, false); - if (destStatusOp.getResult().getStatusCode() == HttpURLConnection.HTTP_OK) { - String lmt = destStatusOp.getResult().getResponseHeader( - HttpHeaderConfigurations.LAST_MODIFIED); - - if (DateTimeUtils.isRecentlyModified(lmt, renameRequestStartTime)) { - return destStatusOp; + try { + final AbfsRestOperation destStatusOp = getPathStatus(destination, + false); + if (destStatusOp.getResult().getStatusCode() + == HttpURLConnection.HTTP_OK) { + String lmt = destStatusOp.getResult().getResponseHeader( + HttpHeaderConfigurations.LAST_MODIFIED); + + if (DateTimeUtils.isRecentlyModified(lmt, renameRequestStartTime)) { + return destStatusOp; + } } + } catch (AzureBlobFileSystemException e) { + // GetFileStatus on the destination failed, return original op + return op; } } @@ -570,10 +586,18 @@ public AbfsRestOperation deletePath(final String path, final boolean recursive, HTTP_METHOD_DELETE, url, requestHeaders); + try { op.execute(); - - if (op.getResult().getStatusCode() != HttpURLConnection.HTTP_OK) { - return deleteIdempotencyCheckOp(op); + } catch (AzureBlobFileSystemException e) { + final AbfsRestOperation idempotencyOp = deleteIdempotencyCheckOp(op); + if (idempotencyOp.getResult().getStatusCode() + == op.getResult().getStatusCode()) { + // idempotency did not return different result + // throw back the exception + throw e; + } else { + return idempotencyOp; + } } return op; @@ -822,7 +846,8 @@ private URL createRequestUrl(final String query) throws AzureBlobFileSystemExcep return createRequestUrl(EMPTY_STRING, query); } - private URL createRequestUrl(final String path, final String query) + @VisibleForTesting + protected URL createRequestUrl(final String path, final String query) throws AzureBlobFileSystemException { final String base = baseUrl.toString(); String encodedPath = path; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index f3986d4b1f35d..936267aa50735 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -24,6 +24,7 @@ import java.net.UnknownHostException; import java.util.List; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -170,7 +171,8 @@ String getSasToken() { * Executes the REST operation with retry, by issuing one or more * HTTP operations. */ - void execute() throws AzureBlobFileSystemException { + @VisibleForTesting + public void execute() throws AzureBlobFileSystemException { // see if we have latency reports from the previous requests String latencyHeader = this.client.getAbfsPerfTracker().getClientLatency(); if (latencyHeader != null && !latencyHeader.isEmpty()) { @@ -181,8 +183,9 @@ void execute() throws AzureBlobFileSystemException { retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); - while (!executeHttpOperation(retryCount++)) { + while (!executeHttpOperation(retryCount)) { try { + ++retryCount; LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); Thread.sleep(client.getRetryPolicy().getRetryInterval(retryCount)); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index e2973968e22fe..2f2a6191ed48a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -30,6 +30,7 @@ import org.junit.Assume; import org.junit.Test; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; @@ -38,9 +39,12 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -168,7 +172,8 @@ public void testDeleteIdempotency() throws Exception { // Set retryCount to non-zero when(op.isARetriedRequest()).thenReturn(true); - // Mock instance of Http Operation response. This will return HTTP:Not Found + // Case 1: Mock instance of Http Operation response. This will return + // HTTP:Not Found AbfsHttpOperation http404Op = mock(AbfsHttpOperation.class); when(http404Op.getStatusCode()).thenReturn(HTTP_NOT_FOUND); @@ -181,6 +186,64 @@ public void testDeleteIdempotency() throws Exception { .describedAs( "Delete is considered idempotent by default and should return success.") .isEqualTo(HTTP_OK); + + // Case 2: Mock instance of Http Operation response. This will return + // HTTP:Bad Request + AbfsHttpOperation http400Op = mock(AbfsHttpOperation.class); + when(http400Op.getStatusCode()).thenReturn(HTTP_BAD_REQUEST); + + // Mock delete response to 400 + when(op.getResult()).thenReturn(http400Op); + + Assertions.assertThat(testClient.deleteIdempotencyCheckOp(op) + .getResult() + .getStatusCode()) + .describedAs( + "Idempotency check to happen only for HTTP 404 response.") + .isEqualTo(HTTP_BAD_REQUEST); + + } + + @Test + public void testDeleteIdempotencyTriggerHttp404() throws Exception { + + final AzureBlobFileSystem fs = getFileSystem(); + AbfsClient client = TestAbfsClient.createTestClientFromCurrentContext( + fs.getAbfsStore().getClient(), + this.getConfiguration()); + + // Case 1: Not a retried case should throw error back + intercept(AbfsRestOperationException.class, + () -> client.deletePath( + "/NonExistingPath", + false, + null)); + + // mock idempotency check to mimic retried case + AbfsClient mockClient = TestAbfsClient.getMockAbfsClient( + fs.getAbfsStore().getClient(), + this.getConfiguration()); + + // Case 2: Mimic retried case + // Idempotency check on Delete always returns success + AbfsRestOperation idempotencyRetOp = mock(AbfsRestOperation.class); + AbfsHttpOperation http200Op = mock(AbfsHttpOperation.class); + when(http200Op.getStatusCode()).thenReturn(HTTP_OK); + when(idempotencyRetOp.getResult()).thenReturn(http200Op); + + doReturn(idempotencyRetOp).when(mockClient).deleteIdempotencyCheckOp(any()); + when(mockClient.deletePath("/NonExistingPath", false, + null)).thenCallRealMethod(); + + Assertions.assertThat(mockClient.deletePath( + "/NonExistingPath", + false, + null) + .getResult() + .getStatusCode()) + .describedAs("Idempotency check reports successful " + + "delete. 200OK should be returned") + .isEqualTo(idempotencyRetOp.getResult().getStatusCode()); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java index 7e03ee5bcc297..2adf70ca6457d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java @@ -30,6 +30,7 @@ import org.junit.Test; import org.junit.Assert; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; @@ -42,6 +43,8 @@ import static java.net.HttpURLConnection.HTTP_OK; import static java.util.UUID.randomUUID; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -51,6 +54,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + /** * Test rename operation. */ @@ -77,6 +82,16 @@ public void testEnsureFileIsRenamed() throws Exception { assertPathDoesNotExist(fs, "expected renamed", src); } + @Test + public void testRenameWithPreExistingDestination() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + Path src = path("renameSrc"); + touch(src); + Path dest = path("renameDest"); + touch(dest); + assertRenameOutcome(fs, src, dest, false); + } + @Test public void testRenameFileUnderDir() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); @@ -197,6 +212,59 @@ public void testRenameRetryFailureWithDestOldLMT() throws Exception { + "TimespanForIdentifyingRecentOperationThroughLMT."); } + @Test + public void testRenameIdempotencyTriggerHttpNotFound() throws Exception { + AbfsHttpOperation http404Op = mock(AbfsHttpOperation.class); + when(http404Op.getStatusCode()).thenReturn(HTTP_NOT_FOUND); + + AbfsHttpOperation http200Op = mock(AbfsHttpOperation.class); + when(http200Op.getStatusCode()).thenReturn(HTTP_OK); + + // Check 1 where idempotency check fails to find dest path + // Rename should throw exception + testRenameIdempotencyTriggerChecks(http404Op); + + // Check 2 where idempotency check finds the dest path + // Renam will be successful + testRenameIdempotencyTriggerChecks(http200Op); + } + + private void testRenameIdempotencyTriggerChecks( + AbfsHttpOperation idempotencyRetHttpOp) throws Exception { + + final AzureBlobFileSystem fs = getFileSystem(); + AbfsClient client = TestAbfsClient.getMockAbfsClient( + fs.getAbfsStore().getClient(), + this.getConfiguration()); + + AbfsRestOperation idempotencyRetOp = mock(AbfsRestOperation.class); + when(idempotencyRetOp.getResult()).thenReturn(idempotencyRetHttpOp); + doReturn(idempotencyRetOp).when(client).renameIdempotencyCheckOp(any(), + any(), any()); + when(client.renamePath(any(), any(), any())).thenCallRealMethod(); + + // rename on non-existing source file will trigger idempotency check + if (idempotencyRetHttpOp.getStatusCode() == HTTP_OK) { + // idempotency check found that destination exists and is recently created + Assertions.assertThat(client.renamePath( + "/NonExistingsourcepath", + "/destpath", + null) + .getResult() + .getStatusCode()) + .describedAs("Idempotency check reports recent successful " + + "rename. 200OK should be returned") + .isEqualTo(idempotencyRetOp.getResult().getStatusCode()); + } else { + // rename dest not found. Original exception should be returned. + intercept(AbfsRestOperationException.class, + () -> client.renamePath( + "/NonExistingsourcepath", + "/destpath", + "")); + } + } + private void testRenameTimeout( int renameRequestStatus, int renameIdempotencyCheckStatus, diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 8197e7e20209e..bab02c09c7423 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.IOException; +import java.lang.reflect.Field; import java.net.MalformedURLException; import java.net.URL; import java.util.regex.Pattern; @@ -33,6 +34,9 @@ import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.APN_VERSION; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CLIENT_VERSION; @@ -271,4 +275,76 @@ public static AbfsClient createTestClientFromCurrentContext( return testClient; } + + public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance, + AbfsConfiguration abfsConfig) + throws IOException, NoSuchFieldException, IllegalAccessException { + AuthType currentAuthType = abfsConfig.getAuthType( + abfsConfig.getAccountName()); + + org.junit.Assume.assumeTrue( + (currentAuthType == AuthType.SharedKey) + || (currentAuthType == AuthType.OAuth)); + + AbfsClient client = mock(AbfsClient.class); + AbfsPerfTracker tracker = new AbfsPerfTracker( + "test", + abfsConfig.getAccountName(), + abfsConfig); + + when(client.getAbfsPerfTracker()).thenReturn(tracker); + when(client.getAuthType()).thenReturn(currentAuthType); + when(client.getRetryPolicy()).thenReturn( + new ExponentialRetryPolicy(1)); + + when(client.createDefaultUriQueryBuilder()).thenCallRealMethod(); + when(client.createRequestUrl(any(), any())).thenCallRealMethod(); + when(client.getAccessToken()).thenCallRealMethod(); + when(client.getSharedKeyCredentials()).thenCallRealMethod(); + when(client.createDefaultHeaders()).thenCallRealMethod(); + + // override baseurl + Field baseUrlField = AbfsClient.class.getDeclaredField("baseUrl"); + baseUrlField.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(baseUrlField, baseUrlField.getModifiers() & ~java.lang.reflect.Modifier.FINAL); + baseUrlField.set(client, baseAbfsClientInstance.getBaseUrl()); + + // override auth provider + if (currentAuthType == AuthType.SharedKey) { + Field sharedKeyCredsField = AbfsClient.class.getDeclaredField( + "sharedKeyCredentials"); + sharedKeyCredsField.setAccessible(true); + modifiersField.setInt(sharedKeyCredsField, + sharedKeyCredsField.getModifiers() + & ~java.lang.reflect.Modifier.FINAL); + sharedKeyCredsField.set(client, new SharedKeyCredentials( + abfsConfig.getAccountName().substring(0, + abfsConfig.getAccountName().indexOf(DOT)), + abfsConfig.getStorageAccountKey())); + } else { + Field tokenProviderField = AbfsClient.class.getDeclaredField( + "tokenProvider"); + tokenProviderField.setAccessible(true); + modifiersField.setInt(tokenProviderField, + tokenProviderField.getModifiers() + & ~java.lang.reflect.Modifier.FINAL); + tokenProviderField.set(client, abfsConfig.getTokenProvider()); + } + + // override user agent + String userAgent = "APN/1.0 Azure Blob FS/3.4.0-SNAPSHOT (PrivateBuild " + + "JavaJRE 1.8.0_252; Linux 5.3.0-59-generic/amd64; openssl-1.0; " + + "UNKNOWN/UNKNOWN) MSFT"; + Field userAgentField = AbfsClient.class.getDeclaredField( + "userAgent"); + userAgentField.setAccessible(true); + modifiersField.setInt(userAgentField, + userAgentField.getModifiers() + & ~java.lang.reflect.Modifier.FINAL); + userAgentField.set(client, userAgent); + + return client; + } } From 1b29c9bfeee0035dd042357038b963843169d44c Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Wed, 22 Jul 2020 13:40:20 +0900 Subject: [PATCH 006/335] HADOOP-17138. Fix spotbugs warnings surfaced after upgrade to 4.0.6. (#2155) --- .../dev-support/findbugs-exclude.xml | 4 +-- .../java/org/apache/hadoop/ipc/Server.java | 4 +-- .../checker/DatasetVolumeChecker.java | 36 +++++++++++-------- .../checker/ThrottledAsyncChecker.java | 2 +- .../hdfs/server/namenode/FSEditLogLoader.java | 2 +- .../dev-support/findbugs-exclude.xml | 14 +++++++- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 2 +- .../dev-support/findbugs-exclude.xml | 6 ++++ .../storage/TestTimelineReaderHBaseDown.java | 7 ++-- 9 files changed, 50 insertions(+), 27 deletions(-) diff --git a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml index e647e678a07a6..f8c3472640f25 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml +++ b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml @@ -16,8 +16,8 @@ --> - + - h_LIB + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 907d55f9be347..cc5d941903f8c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -3714,7 +3714,7 @@ void incrUserConnections(String user) { if (count == null) { count = 1; } else { - count++; + count = count + 1; } userToConnectionsMap.put(user, count); } @@ -3726,7 +3726,7 @@ void decrUserConnections(String user) { if (count == null) { return; } else { - count--; + count = count - 1; } if (count == 0) { userToConnectionsMap.remove(user); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index 91582fe0558a8..e31ac6f1241ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -354,23 +354,29 @@ private class ResultHandler } @Override - public void onSuccess(@Nonnull VolumeCheckResult result) { - switch(result) { - case HEALTHY: - case DEGRADED: - LOG.debug("Volume {} is {}.", reference.getVolume(), result); - markHealthy(); - break; - case FAILED: - LOG.warn("Volume {} detected as being unhealthy", + public void onSuccess(VolumeCheckResult result) { + if (result == null) { + LOG.error("Unexpected health check result null for volume {}", reference.getVolume()); - markFailed(); - break; - default: - LOG.error("Unexpected health check result {} for volume {}", - result, reference.getVolume()); markHealthy(); - break; + } else { + switch(result) { + case HEALTHY: + case DEGRADED: + LOG.debug("Volume {} is {}.", reference.getVolume(), result); + markHealthy(); + break; + case FAILED: + LOG.warn("Volume {} detected as being unhealthy", + reference.getVolume()); + markFailed(); + break; + default: + LOG.error("Unexpected health check result {} for volume {}", + result, reference.getVolume()); + markHealthy(); + break; + } } cleanup(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java index 6f04129b4928c..610c8fd97e83b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java @@ -166,7 +166,7 @@ private void addResultCachingCallback( Checkable target, ListenableFuture lf) { Futures.addCallback(lf, new FutureCallback() { @Override - public void onSuccess(@Nullable V result) { + public void onSuccess(V result) { synchronized (ThrottledAsyncChecker.this) { checksInProgress.remove(target); completedChecks.put(target, new LastCheckResult<>( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index e3694ba4f4ca8..280e070950cea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -1238,7 +1238,7 @@ private void incrOpCount(FSEditLogOpCodes opCode, holder = new Holder(1); opCounts.put(opCode, holder); } else { - holder.held++; + holder.held = holder.held + 1; } counter.increment(); } diff --git a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml index 9b4d8c90f5027..4e459b652b29c 100644 --- a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml +++ b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml @@ -533,5 +533,17 @@ - + + + + + + + + + + diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 6f75bd17c6f0a..5bfa8dc021a34 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -813,7 +813,7 @@ private void increaseQueueAppNum(String queue) throws YarnException { if (appNum == null) { appNum = 1; } else { - appNum++; + appNum = appNum + 1; } queueAppNumMap.put(queueName, appNum); diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 3a37293357a01..95706f9e1360c 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -705,4 +705,10 @@ + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java index 1148b80d1962a..d83f130338109 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java @@ -181,14 +181,13 @@ private static void waitForHBaseDown(HBaseTimelineReaderImpl htr) throws } } - private static void checkQuery(HBaseTimelineReaderImpl htr) throws - IOException { + private static Set checkQuery(HBaseTimelineReaderImpl htr) + throws IOException { TimelineReaderContext context = new TimelineReaderContext(YarnConfiguration.DEFAULT_RM_CLUSTER_ID, null, null, null, null, TimelineEntityType .YARN_FLOW_ACTIVITY.toString(), null, null); - Set entities = htr.getEntities(context, MONITOR_FILTERS, - DATA_TO_RETRIEVE); + return htr.getEntities(context, MONITOR_FILTERS, DATA_TO_RETRIEVE); } private static void configure(HBaseTestingUtility util) { From ac9a07b51aefd0fd3b4602adc844ab0f172835e3 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Tue, 21 Jul 2020 23:29:10 -0700 Subject: [PATCH 007/335] HDFS-15478: When Empty mount points, we are assigning fallback link to self. But it should not use full URI for target fs. (#2160). Contributed by Uma Maheswara Rao G. --- .../hadoop/fs/viewfs/ViewFileSystem.java | 2 +- .../TestViewFsOverloadSchemeListStatus.java | 27 +++++++++++++------ .../src/site/markdown/ViewFsOverloadScheme.md | 2 ++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 1fc531e05635d..baf0027fa6a84 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -294,7 +294,7 @@ public void initialize(final URI theUri, final Configuration conf) myUri = new URI(getScheme(), authority, "/", null, null); boolean initingUriAsFallbackOnNoMounts = !FsConstants.VIEWFS_TYPE.equals(getType()); - fsState = new InodeTree(conf, tableName, theUri, + fsState = new InodeTree(conf, tableName, myUri, initingUriAsFallbackOnNoMounts) { @Override protected FileSystem getTargetFileSystem(final URI uri) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java index 300fdd8b333f1..7afc78981f6e3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java @@ -127,19 +127,30 @@ public void testListStatusACL() throws IOException, URISyntaxException { /** * Tests that ViewFSOverloadScheme should consider initialized fs as fallback - * if there are no mount links configured. + * if there are no mount links configured. It should add fallback with the + * chrootedFS at it's uri's root. */ @Test(timeout = 30000) public void testViewFSOverloadSchemeWithoutAnyMountLinks() throws Exception { - try (FileSystem fs = FileSystem.get(TEST_DIR.toPath().toUri(), conf)) { + Path initUri = new Path(TEST_DIR.toURI().toString(), "init"); + try (FileSystem fs = FileSystem.get(initUri.toUri(), conf)) { ViewFileSystemOverloadScheme vfs = (ViewFileSystemOverloadScheme) fs; assertEquals(0, vfs.getMountPoints().length); - Path testFallBack = new Path("test", FILE_NAME); - assertTrue(vfs.mkdirs(testFallBack)); - FileStatus[] status = vfs.listStatus(testFallBack.getParent()); - assertEquals(FILE_NAME, status[0].getPath().getName()); - assertEquals(testFallBack.getName(), - vfs.getFileLinkStatus(testFallBack).getPath().getName()); + Path testOnFallbackPath = new Path(TEST_DIR.toURI().toString(), "test"); + assertTrue(vfs.mkdirs(testOnFallbackPath)); + FileStatus[] status = vfs.listStatus(testOnFallbackPath.getParent()); + assertEquals(Path.getPathWithoutSchemeAndAuthority(testOnFallbackPath), + Path.getPathWithoutSchemeAndAuthority(status[0].getPath())); + //Check directly on localFS. The fallBackFs(localFS) should be chrooted + //at it's root. So, after + FileSystem lfs = vfs.getRawFileSystem(testOnFallbackPath, conf); + FileStatus[] statusOnLocalFS = + lfs.listStatus(testOnFallbackPath.getParent()); + assertEquals(testOnFallbackPath.getName(), + statusOnLocalFS[0].getPath().getName()); + //initUri should not have exist in lfs, as it would have chrooted on it's + // root only. + assertFalse(lfs.exists(initUri)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md index 564bc034e7597..f3eb336da6121 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md @@ -34,6 +34,8 @@ If a user wants to continue use the same fs.defaultFS and wants to have more mou Example if fs.defaultFS is `hdfs://mycluster`, then the mount link configuration key name should be like in the following format `fs.viewfs.mounttable.*mycluster*.link.`. Even if the initialized fs uri has hostname:port, it will simply ignore the port number and only consider the hostname as the mount table name. We will discuss more example configurations in following sections. If there are no mount links configured with the initializing uri's hostname as the mount table name, then it will automatically consider the current uri as fallback(`fs.viewfs.mounttable.*mycluster*.linkFallback`) target fs uri. +If the initialized uri contains path part, it will consider only scheme and authority part, but not the path part. Example, if the initialized uri contains `hdfs://mycluster/data`, it will consider only `hdfs://mycluster` as fallback target fs uri. +The path part `data` will be ignored. Another important improvement with the ViewFileSystemOverloadScheme is, administrators need not copy the `mount-table.xml` configuration file to 1000s of client nodes. Instead, they can keep the mount-table configuration file in a Hadoop compatible file system. So, keeping the configuration file in a central place makes administrators life easier as they can update mount-table in single place. From d5b476615820a7fa75b41e323db5deb5c2ed3bd5 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 23 Jul 2020 00:39:11 +0900 Subject: [PATCH 008/335] HADOOP-17147. Dead link in hadoop-kms/index.md.vm. Contributed by Xieming Li. --- hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm index 5490219750020..656e1cf534abf 100644 --- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm +++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm @@ -307,7 +307,7 @@ Configure `etc/hadoop/ssl-server.xml` with proper values, for example: ``` The SSL passwords can be secured by a credential provider. See -[Credential Provider API](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). +[Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). You need to create an SSL certificate for the KMS. As the `kms` Unix user, using the Java `keytool` command to create the SSL certificate: From 48a7c5b6baf3cbf5ef85433c348753842eb8ec7d Mon Sep 17 00:00:00 2001 From: Mehakmeet Singh Date: Wed, 22 Jul 2020 22:52:30 +0530 Subject: [PATCH 009/335] HADOOP-17113. Adding ReadAhead Counters in ABFS (#2154) Contributed by Mehakmeet Singh --- .../fs/azurebfs/services/AbfsInputStream.java | 6 ++ .../services/AbfsInputStreamStatistics.java | 12 +++ .../AbfsInputStreamStatisticsImpl.java | 36 +++++++ .../ITestAbfsInputStreamStatistics.java | 95 +++++++++++++++++++ 4 files changed, 149 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index a809bde6c3035..926c23d7c53b6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -238,6 +238,9 @@ private int readInternal(final long position, final byte[] b, final int offset, if (receivedBytes > 0) { incrementReadOps(); LOG.debug("Received data from read ahead, not doing remote read"); + if (streamStatistics != null) { + streamStatistics.readAheadBytesRead(receivedBytes); + } return receivedBytes; } @@ -292,6 +295,9 @@ int readRemote(long position, byte[] b, int offset, int length) throws IOExcepti throw new IOException(ex); } long bytesRead = op.getResult().getBytesReceived(); + if (streamStatistics != null) { + streamStatistics.remoteBytesRead(bytesRead); + } if (bytesRead > Integer.MAX_VALUE) { throw new IOException("Unexpected Content-Length"); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatistics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatistics.java index 2603394c9337f..c910a1f75e02e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatistics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatistics.java @@ -84,6 +84,18 @@ public interface AbfsInputStreamStatistics { */ void remoteReadOperation(); + /** + * Records the bytes read from readAhead buffer. + * @param bytes the bytes to be incremented. + */ + void readAheadBytesRead(long bytes); + + /** + * Records bytes read remotely after nothing from readAheadBuffer was read. + * @param bytes the bytes to be incremented. + */ + void remoteBytesRead(long bytes); + /** * Makes the string of all the AbfsInputStream statistics. * @return the string with all the statistics. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatisticsImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatisticsImpl.java index fd18910813d39..12cc407dcbcf0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatisticsImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamStatisticsImpl.java @@ -33,6 +33,8 @@ public class AbfsInputStreamStatisticsImpl private long readOperations; private long bytesReadFromBuffer; private long remoteReadOperations; + private long readAheadBytesRead; + private long remoteBytesRead; /** * Seek backwards, incrementing the seek and backward seek counters. @@ -128,6 +130,30 @@ public void readOperationStarted(long pos, long len) { readOperations++; } + /** + * Total bytes read from readAhead buffer during a read operation. + * + * @param bytes the bytes to be incremented. + */ + @Override + public void readAheadBytesRead(long bytes) { + if (bytes > 0) { + readAheadBytesRead += bytes; + } + } + + /** + * Total bytes read remotely after nothing was read from readAhead buffer. + * + * @param bytes the bytes to be incremented. + */ + @Override + public void remoteBytesRead(long bytes) { + if (bytes > 0) { + remoteBytesRead += bytes; + } + } + /** * {@inheritDoc} * @@ -178,6 +204,14 @@ public long getRemoteReadOperations() { return remoteReadOperations; } + public long getReadAheadBytesRead() { + return readAheadBytesRead; + } + + public long getRemoteBytesRead() { + return remoteBytesRead; + } + /** * String operator describes all the current statistics. * Important: there are no guarantees as to the stability @@ -199,6 +233,8 @@ public String toString() { sb.append(", ReadOperations=").append(readOperations); sb.append(", bytesReadFromBuffer=").append(bytesReadFromBuffer); sb.append(", remoteReadOperations=").append(remoteReadOperations); + sb.append(", readAheadBytesRead=").append(readAheadBytesRead); + sb.append(", remoteBytesRead=").append(remoteBytesRead); sb.append('}'); return sb.toString(); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java index 7a62ecab7f4ea..8385099a78d36 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java @@ -20,6 +20,7 @@ import java.io.IOException; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,10 @@ public class ITestAbfsInputStreamStatistics LoggerFactory.getLogger(ITestAbfsInputStreamStatistics.class); private static final int ONE_MB = 1024 * 1024; private static final int ONE_KB = 1024; + private static final int CUSTOM_BLOCK_BUFFER_SIZE = 4 * 1024; + private static final int CUSTOM_READ_AHEAD_BUFFER_SIZE = 8 * CUSTOM_BLOCK_BUFFER_SIZE; + private static final int THREAD_SLEEP_10_SECONDS = 10; + private static final int TIMEOUT_30_SECONDS = 30000; private byte[] defBuffer = new byte[ONE_MB]; public ITestAbfsInputStreamStatistics() throws Exception { @@ -75,6 +80,8 @@ public void testInitValues() throws IOException { checkInitValue(stats.getReadOperations(), "readOps"); checkInitValue(stats.getBytesReadFromBuffer(), "bytesReadFromBuffer"); checkInitValue(stats.getRemoteReadOperations(), "remoteReadOps"); + checkInitValue(stats.getReadAheadBytesRead(), "readAheadBytesRead"); + checkInitValue(stats.getRemoteBytesRead(), "readAheadRemoteBytesRead"); } finally { IOUtils.cleanupWithLogger(LOG, outputStream, inputStream); @@ -285,6 +292,94 @@ public void testWithNullStreamStatistics() throws IOException { } } + /** + * Testing readAhead counters in AbfsInputStream with 30 seconds timeout. + */ + @Test(timeout = TIMEOUT_30_SECONDS) + public void testReadAheadCounters() throws IOException, InterruptedException { + describe("Test to check correct values for readAhead counters in " + + "AbfsInputStream"); + + AzureBlobFileSystem fs = getFileSystem(); + AzureBlobFileSystemStore abfss = fs.getAbfsStore(); + Path readAheadCountersPath = path(getMethodName()); + + /* + * Setting the block size for readAhead as 4KB. + */ + abfss.getAbfsConfiguration().setReadBufferSize(CUSTOM_BLOCK_BUFFER_SIZE); + + AbfsOutputStream out = null; + AbfsInputStream in = null; + + try { + + /* + * Creating a file of 1MB size. + */ + out = createAbfsOutputStreamWithFlushEnabled(fs, readAheadCountersPath); + out.write(defBuffer); + out.close(); + + in = abfss.openFileForRead(readAheadCountersPath, fs.getFsStatistics()); + + /* + * Reading 1KB after each i * KB positions. Hence the reads are from 0 + * to 1KB, 1KB to 2KB, and so on.. for 5 operations. + */ + for (int i = 0; i < 5; i++) { + in.seek(ONE_KB * i); + in.read(defBuffer, ONE_KB * i, ONE_KB); + } + AbfsInputStreamStatisticsImpl stats = + (AbfsInputStreamStatisticsImpl) in.getStreamStatistics(); + + /* + * Since, readAhead is done in background threads. Sometimes, the + * threads aren't finished in the background and could result in + * inaccurate results. So, we wait till we have the accurate values + * with a limit of 30 seconds as that's when the test times out. + * + */ + while (stats.getRemoteBytesRead() < CUSTOM_READ_AHEAD_BUFFER_SIZE + || stats.getReadAheadBytesRead() < CUSTOM_BLOCK_BUFFER_SIZE) { + Thread.sleep(THREAD_SLEEP_10_SECONDS); + } + + /* + * Verifying the counter values of readAheadBytesRead and remoteBytesRead. + * + * readAheadBytesRead : Since, we read 1KBs 5 times, that means we go + * from 0 to 5KB in the file. The bufferSize is set to 4KB, and since + * we have 8 blocks of readAhead buffer. We would have 8 blocks of 4KB + * buffer. Our read is till 5KB, hence readAhead would ideally read 2 + * blocks of 4KB which is equal to 8KB. But, sometimes to get more than + * one block from readAhead buffer we might have to wait for background + * threads to fill the buffer and hence we might do remote read which + * would be faster. Therefore, readAheadBytesRead would be equal to or + * greater than 4KB. + * + * remoteBytesRead : Since, the bufferSize is set to 4KB and the number + * of blocks or readAheadQueueDepth is equal to 8. We would read 8 * 4 + * KB buffer on the first read, which is equal to 32KB. But, if we are not + * able to read some bytes that were in the buffer after doing + * readAhead, we might use remote read again. Thus, the bytes read + * remotely could also be greater than 32Kb. + * + */ + Assertions.assertThat(stats.getReadAheadBytesRead()).describedAs( + "Mismatch in readAheadBytesRead counter value") + .isGreaterThanOrEqualTo(CUSTOM_BLOCK_BUFFER_SIZE); + + Assertions.assertThat(stats.getRemoteBytesRead()).describedAs( + "Mismatch in remoteBytesRead counter value") + .isGreaterThanOrEqualTo(CUSTOM_READ_AHEAD_BUFFER_SIZE); + + } finally { + IOUtils.cleanupWithLogger(LOG, out, in); + } + } + /** * Method to assert the initial values of the statistics. * From 2d12496643b1b7cfa4eb270ec9b2fcdb78a58798 Mon Sep 17 00:00:00 2001 From: bshashikant Date: Thu, 23 Jul 2020 04:46:27 +0530 Subject: [PATCH 010/335] HDFS-15480. Ordered snapshot deletion: record snapshot deletion in XAttr (#2163) --- .../server/common/HdfsServerConstants.java | 1 + .../hdfs/server/namenode/FSDirSnapshotOp.java | 20 --- .../hdfs/server/namenode/FSDirXAttrOp.java | 13 +- .../hdfs/server/namenode/FSDirectory.java | 10 +- .../namenode/snapshot/SnapshotManager.java | 50 +++++- .../namenode/TestOrderedSnapshotDeletion.java | 157 +++++++++++++++--- 6 files changed, 186 insertions(+), 65 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 78d4289a047cb..a55985edffcc5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -366,6 +366,7 @@ enum BlockUCState { "security.hdfs.unreadable.by.superuser"; String XATTR_ERASURECODING_POLICY = "system.hdfs.erasurecoding.policy"; + String SNAPSHOT_XATTR_NAME = "system.hdfs.snapshot.deleted"; String XATTR_SATISFY_STORAGE_POLICY = "user.hdfs.sps"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java index c2eb401c5fbb8..923c6a88b0318 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java @@ -21,7 +21,6 @@ import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.FSLimitException; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -252,25 +251,6 @@ static INode.BlocksMapUpdateInfo deleteSnapshot( // time of snapshot deletion final long now = Time.now(); - if (fsd.isSnapshotDeletionOrdered()) { - final INodeDirectory srcRoot = snapshotManager.getSnapshottableRoot(iip); - final DirectorySnapshottableFeature snapshottable - = srcRoot.getDirectorySnapshottableFeature(); - final Snapshot snapshot = snapshottable.getSnapshotByName( - srcRoot, snapshotName); - - // Diffs must be not empty since a snapshot exists in the list - final int earliest = snapshottable.getDiffs().iterator().next() - .getSnapshotId(); - if (snapshot.getId() != earliest) { - throw new SnapshotException("Failed to delete snapshot " + snapshotName - + " from directory " + srcRoot.getFullPathName() - + ": " + snapshot + " is not the earliest snapshot id=" + earliest - + " (" + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED - + " is " + fsd.isSnapshotDeletionOrdered() + ")"); - } - } - final INode.BlocksMapUpdateInfo collectedBlocks = deleteSnapshot( fsd, snapshotManager, iip, snapshotName, now); fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index ff82610f545bb..4f215acda1ba4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -41,12 +41,13 @@ import java.util.List; import java.util.ListIterator; -import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SNAPSHOT_XATTR_NAME; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE; -class FSDirXAttrOp { +public class FSDirXAttrOp { private static final XAttr KEYID_XATTR = XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null); private static final XAttr UNREADABLE_BY_SUPERUSER_XATTR = @@ -265,7 +266,7 @@ static List filterINodeXAttrs( return newXAttrs; } - static INode unprotectedSetXAttrs( + public static INode unprotectedSetXAttrs( FSDirectory fsd, final INodesInPath iip, final List xAttrs, final EnumSet flag) throws IOException { @@ -326,6 +327,12 @@ static INode unprotectedSetXAttrs( throw new IOException("Can only set '" + SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' on a file."); } + + if (xaName.equals(SNAPSHOT_XATTR_NAME) && !(inode.isDirectory() && + inode.getParent().isSnapshottable())) { + throw new IOException("Can only set '" + + SNAPSHOT_XATTR_NAME + "' on a snapshot root."); + } } XAttrStorage.updateINodeXAttrs(inode, newXAttrs, iip.getLatestSnapshotId()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 6df255e86574e..2c7932c0f8394 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -359,14 +359,6 @@ public enum DirOp { DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT); LOG.info("{} = {}", DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, snapshotDeletionOrdered); - if (snapshotDeletionOrdered && !xattrsEnabled) { - throw new HadoopIllegalArgumentException("" + - "XAttrs is required by snapshotDeletionOrdered:" - + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED - + " is true but " - + DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY - + " is false."); - } this.accessTimePrecision = conf.getLong( DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, @@ -626,7 +618,7 @@ boolean isXattrsEnabled() { } int getXattrMaxSize() { return xattrMaxSize; } - boolean isSnapshotDeletionOrdered() { + public boolean isSnapshotDeletionOrdered() { return snapshotDeletionOrdered; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 30b98b8e86421..9ace8a97b8641 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -36,10 +36,14 @@ import javax.management.ObjectName; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; @@ -47,14 +51,9 @@ import org.apache.hadoop.hdfs.protocol.SnapshotInfo; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; -import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.namenode.*; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; -import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; -import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.server.namenode.INode; -import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; -import org.apache.hadoop.hdfs.server.namenode.INodesInPath; -import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.metrics2.util.MBeans; import com.google.common.base.Preconditions; @@ -352,7 +351,38 @@ public String createSnapshot(final LeaseManager leaseManager, */ public void deleteSnapshot(final INodesInPath iip, final String snapshotName, INode.ReclaimContext reclaimContext, long now) throws IOException { - INodeDirectory srcRoot = getSnapshottableRoot(iip); + final INodeDirectory srcRoot = getSnapshottableRoot(iip); + if (fsdir.isSnapshotDeletionOrdered()) { + final DirectorySnapshottableFeature snapshottable + = srcRoot.getDirectorySnapshottableFeature(); + final Snapshot snapshot = snapshottable.getSnapshotByName( + srcRoot, snapshotName); + + // Diffs must be not empty since a snapshot exists in the list + final int earliest = snapshottable.getDiffs().iterator().next() + .getSnapshotId(); + if (snapshot.getId() != earliest) { + final XAttr snapshotXAttr = buildXAttr(); + final List xattrs = Lists.newArrayListWithCapacity(1); + xattrs.add(snapshotXAttr); + + // The snapshot to be deleted is just marked for deletion in the xAttr. + // Same snaphot delete call can happen multiple times until and unless + // the very 1st instance of a snapshot delete hides it/remove it from + // snapshot list. XAttrSetFlag.REPLACE needs to be set to here in order + // to address this. + + // XAttr will set on the snapshot root directory + // NOTE : This function is directly called while replaying the edit + // logs.While replaying the edit logs we need to mark the snapshot + // deleted in the xattr of the snapshot root. + FSDirXAttrOp.unprotectedSetXAttrs(fsdir, + INodesInPath.append(iip, snapshot.getRoot(), + DFSUtil.string2Bytes(snapshotName)), xattrs, + EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); + return; + } + } srcRoot.removeSnapshot(reclaimContext, snapshotName, now); numSnapshots.getAndDecrement(); } @@ -545,6 +575,10 @@ public int getMaxSnapshotID() { return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 1); } + public static XAttr buildXAttr() { + return XAttrHelper.buildXAttr(HdfsServerConstants.SNAPSHOT_XATTR_NAME); + } + private ObjectName mxBeanName; public void registerMXBean() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java index c8df780465bab..d3097ea507770 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java @@ -19,32 +19,35 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.SnapshotException; +import org.apache.hadoop.hdfs.XAttrHelper; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; -import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.event.Level; import java.io.IOException; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Map; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; +import static org.apache.hadoop.hdfs.DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; -/** Test ordered snapshot deletion. */ +/** + * Test ordered snapshot deletion. + */ public class TestOrderedSnapshotDeletion { - static final Logger LOG = LoggerFactory.getLogger(FSDirectory.class); - - { - SnapshotTestHelper.disableLogs(); - GenericTestUtils.setLogLevel(INode.LOG, Level.TRACE); - } - + static final String xattrName = "user.a1"; + static final byte[] xattrValue = {0x31, 0x32, 0x33}; private final Path snapshottableDir = new Path("/" + getClass().getSimpleName()); @@ -67,7 +70,7 @@ public void tearDown() throws Exception { } } - @Test (timeout=60000) + @Test(timeout = 60000) public void testConf() throws Exception { DistributedFileSystem hdfs = cluster.getFileSystem(); hdfs.mkdirs(snapshottableDir); @@ -85,21 +88,125 @@ public void testConf() throws Exception { hdfs.mkdirs(sub2); hdfs.createSnapshot(snapshottableDir, "s2"); - assertDeletionDenied(snapshottableDir, "s1", hdfs); - assertDeletionDenied(snapshottableDir, "s2", hdfs); + assertXAttrSet("s1", hdfs, null); + assertXAttrSet("s2", hdfs, null); hdfs.deleteSnapshot(snapshottableDir, "s0"); - assertDeletionDenied(snapshottableDir, "s2", hdfs); + assertXAttrSet("s2", hdfs, null); hdfs.deleteSnapshot(snapshottableDir, "s1"); hdfs.deleteSnapshot(snapshottableDir, "s2"); } - static void assertDeletionDenied(Path snapshottableDir, String snapshot, - DistributedFileSystem hdfs) throws IOException { + void assertXAttrSet(String snapshot, + DistributedFileSystem hdfs, XAttr newXattr) + throws IOException { + hdfs.deleteSnapshot(snapshottableDir, snapshot); + // Check xAttr for parent directory + FSNamesystem namesystem = cluster.getNamesystem(); + Path snapshotRoot = SnapshotTestHelper.getSnapshotRoot(snapshottableDir, + snapshot); + INode inode = namesystem.getFSDirectory().getINode(snapshotRoot.toString()); + XAttrFeature f = inode.getXAttrFeature(); + XAttr xAttr = f.getXAttr(HdfsServerConstants.SNAPSHOT_XATTR_NAME); + assertTrue("Snapshot xAttr should exist", xAttr != null); + assertTrue(xAttr.getName().equals(HdfsServerConstants.SNAPSHOT_XATTR_NAME. + replace("system.", ""))); + assertTrue(xAttr.getNameSpace().equals(XAttr.NameSpace.SYSTEM)); + assertNull(xAttr.getValue()); + + // Make sure its not user visible + if (cluster.getNameNode().getConf().getBoolean(DFSConfigKeys. + DFS_NAMENODE_XATTRS_ENABLED_KEY, + DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT)) { + Map xattrMap = hdfs.getXAttrs(snapshotRoot); + assertTrue(newXattr == null ? xattrMap.isEmpty() : + Arrays.equals(newXattr.getValue(), xattrMap.get(xattrName))); + } + } + + @Test(timeout = 60000) + public void testSnapshotXattrPersistence() throws Exception { + DistributedFileSystem hdfs = cluster.getFileSystem(); + hdfs.mkdirs(snapshottableDir); + hdfs.allowSnapshot(snapshottableDir); + + final Path sub0 = new Path(snapshottableDir, "sub0"); + hdfs.mkdirs(sub0); + hdfs.createSnapshot(snapshottableDir, "s0"); + + final Path sub1 = new Path(snapshottableDir, "sub1"); + hdfs.mkdirs(sub1); + hdfs.createSnapshot(snapshottableDir, "s1"); + assertXAttrSet("s1", hdfs, null); + assertXAttrSet("s1", hdfs, null); + cluster.restartNameNodes(); + assertXAttrSet("s1", hdfs, null); + } + + @Test(timeout = 60000) + public void testSnapshotXattrWithSaveNameSpace() throws Exception { + DistributedFileSystem hdfs = cluster.getFileSystem(); + hdfs.mkdirs(snapshottableDir); + hdfs.allowSnapshot(snapshottableDir); + + final Path sub0 = new Path(snapshottableDir, "sub0"); + hdfs.mkdirs(sub0); + hdfs.createSnapshot(snapshottableDir, "s0"); + + final Path sub1 = new Path(snapshottableDir, "sub1"); + hdfs.mkdirs(sub1); + hdfs.createSnapshot(snapshottableDir, "s1"); + assertXAttrSet("s1", hdfs, null); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + cluster.restartNameNodes(); + assertXAttrSet("s1", hdfs, null); + } + + @Test(timeout = 60000) + public void testSnapshotXattrWithDisablingXattr() throws Exception { + DistributedFileSystem hdfs = cluster.getFileSystem(); + hdfs.mkdirs(snapshottableDir); + hdfs.allowSnapshot(snapshottableDir); + + final Path sub0 = new Path(snapshottableDir, "sub0"); + hdfs.mkdirs(sub0); + hdfs.createSnapshot(snapshottableDir, "s0"); + + final Path sub1 = new Path(snapshottableDir, "sub1"); + hdfs.mkdirs(sub1); + hdfs.createSnapshot(snapshottableDir, "s1"); + assertXAttrSet("s1", hdfs, null); + cluster.getNameNode().getConf().setBoolean( + DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, false); + cluster.restartNameNodes(); + // ensure xAttr feature is disabled try { - hdfs.deleteSnapshot(snapshottableDir, snapshot); - Assert.fail("deleting " +snapshot + " should fail"); - } catch (SnapshotException se) { - LOG.info("Good, it is expected to have " + se); + hdfs.getXAttrs(snapshottableDir); + } catch (Exception e) { + assertTrue(e.getMessage().contains("The XAttr operation has been " + + "rejected. Support for XAttrs has been disabled by " + + "setting dfs.namenode.xattrs.enabled to false")); } + // try deleting snapshot and verify it still sets the snapshot XAttr + assertXAttrSet("s1", hdfs, null); + } + + @Test(timeout = 60000) + public void testSnapshotXAttrWithPreExistingXattrs() throws Exception { + DistributedFileSystem hdfs = cluster.getFileSystem(); + hdfs.mkdirs(snapshottableDir); + hdfs.allowSnapshot(snapshottableDir); + hdfs.setXAttr(snapshottableDir, xattrName, xattrValue, + EnumSet.of(XAttrSetFlag.CREATE)); + XAttr newXAttr = XAttrHelper.buildXAttr(xattrName, xattrValue); + final Path sub0 = new Path(snapshottableDir, "sub0"); + hdfs.mkdirs(sub0); + hdfs.createSnapshot(snapshottableDir, "s0"); + + final Path sub1 = new Path(snapshottableDir, "sub1"); + hdfs.mkdirs(sub1); + hdfs.createSnapshot(snapshottableDir, "s1"); + assertXAttrSet("s1", hdfs, newXAttr); } } From bfcd775381f1e0b94b17ce3cfca7eade95df1ea8 Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Thu, 23 Jul 2020 16:40:24 +0530 Subject: [PATCH 011/335] YARN-10315. Avoid sending RMNodeResourceupdate event if resource is same. Contributed by Sushil Ks. --- .../resourcemanager/scheduler/AbstractYarnScheduler.java | 4 +++- .../scheduler/capacity/TestCapacityScheduler.java | 9 +++++++++ .../scheduler/fair/TestFairScheduler.java | 9 +++++++++ .../scheduler/fifo/TestFifoScheduler.java | 9 +++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 70d2714c1409c..43d8f3a0dddd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -1188,7 +1188,9 @@ protected void nodeUpdate(RMNode nm) { // If the node is decommissioning, send an update to have the total // resource equal to the used resource, so no available resource to // schedule. - if (nm.getState() == NodeState.DECOMMISSIONING && schedulerNode != null) { + if (nm.getState() == NodeState.DECOMMISSIONING && schedulerNode != null + && schedulerNode.getTotalResource().compareTo( + schedulerNode.getAllocatedResource()) != 0) { this.rmContext .getDispatcher() .getEventHandler() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index a272851cd2bcb..1c7020aec6eef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -38,6 +38,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; @@ -4726,6 +4728,13 @@ public void handle(Event event) { availableResource.getMemorySize()); Assert.assertEquals("Available Resource Memory Size should be 0", 0, availableResource.getVirtualCores()); + // Kick off another heartbeat where the RMNodeResourceUpdateEvent would + // be skipped for DECOMMISSIONING state since the total resource is + // already equal to used resource from the previous heartbeat. + when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); + resourceManager.getResourceScheduler().handle( + new NodeUpdateSchedulerEvent(spyNode)); + verify(mockDispatcher, times(4)).getEventHandler(); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 7882ba3664cd6..78e8e2d1aaa6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -136,6 +136,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @SuppressWarnings("unchecked") @@ -4956,6 +4958,13 @@ public void handle(Event event) { .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource(); assertThat(availableResource.getMemorySize()).isEqualTo(0); assertThat(availableResource.getVirtualCores()).isEqualTo(0); + // Kick off another heartbeat where the RMNodeResourceUpdateEvent would + // be skipped for DECOMMISSIONING state since the total resource is + // already equal to used resource from the previous heartbeat. + when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); + resourceManager.getResourceScheduler().handle( + new NodeUpdateSchedulerEvent(spyNode)); + verify(mockDispatcher, times(1)).getEventHandler(); } private NodeManager registerNode(String hostName, int containerManagerPort, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 9b3657e00d5a2..40650532ed2d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -24,6 +24,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; @@ -1314,6 +1316,13 @@ public void handle(Event event) { .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource(); assertThat(availableResource.getMemorySize()).isEqualTo(0); assertThat(availableResource.getVirtualCores()).isEqualTo(0); + // Kick off another heartbeat where the RMNodeResourceUpdateEvent would + // be skipped for DECOMMISSIONING state since the total resource is + // already equal to used resource from the previous heartbeat. + when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING); + resourceManager.getResourceScheduler().handle( + new NodeUpdateSchedulerEvent(spyNode)); + verify(mockDispatcher, times(4)).getEventHandler(); } private void checkApplicationResourceUsage(int expected, From 247eb0979b6a3a723ea9a249ba4db1ee079eb909 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 16 Jun 2020 21:52:32 +0530 Subject: [PATCH 012/335] YARN-10319. Record Last N Scheduler Activities from ActivitiesManager Reviewed by Tao Yang and Adam Antal. --- .../activities/ActivitiesManager.java | 40 ++- .../webapp/JAXBContextResolver.java | 3 +- .../resourcemanager/webapp/RMWSConsts.java | 5 + .../webapp/RMWebServiceProtocol.java | 14 + .../resourcemanager/webapp/RMWebServices.java | 331 ++++++++++-------- .../webapp/dao/BulkActivitiesInfo.java | 52 +++ .../webapp/ActivitiesTestUtils.java | 2 + .../TestRMWebServicesSchedulerActivities.java | 100 ++++++ .../webapp/DefaultRequestInterceptorREST.java | 10 + .../webapp/FederationInterceptorREST.java | 7 + .../router/webapp/RouterWebServices.java | 19 + .../webapp/BaseRouterWebServicesTest.java | 7 + .../webapp/MockRESTRequestInterceptor.java | 7 + .../PassThroughRESTRequestInterceptor.java | 8 + .../src/site/markdown/ResourceManagerRest.md | 134 +++++++ 15 files changed, 593 insertions(+), 146 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/BulkActivitiesInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java index cc02ff688ccb8..3662a77291f67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -44,8 +45,10 @@ import org.apache.hadoop.yarn.util.SystemClock; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.List; import java.util.Set; import java.util.*; @@ -75,7 +78,7 @@ public class ActivitiesManager extends AbstractService { appsAllocation; @VisibleForTesting ConcurrentMap> completedAppAllocations; - private boolean recordNextAvailableNode = false; + private AtomicInteger recordCount = new AtomicInteger(0); private List lastAvailableNodeActivities = null; private Thread cleanUpThread; private long activitiesCleanupIntervalMs; @@ -86,6 +89,8 @@ public class ActivitiesManager extends AbstractService { private final RMContext rmContext; private volatile boolean stopped; private ThreadLocal diagnosticCollectorManager; + private volatile ConcurrentLinkedDeque>> + lastNActivities; public ActivitiesManager(RMContext rmContext) { super(ActivitiesManager.class.getName()); @@ -102,6 +107,7 @@ public ActivitiesManager(RMContext rmContext) { if (rmContext.getYarnConfiguration() != null) { setupConfForCleanup(rmContext.getYarnConfiguration()); } + lastNActivities = new ConcurrentLinkedDeque<>(); } private void setupConfForCleanup(Configuration conf) { @@ -215,9 +221,30 @@ public ActivitiesInfo getActivitiesInfo(String nodeId, return new ActivitiesInfo(allocations, nodeId, groupBy); } + + public List recordAndGetBulkActivitiesInfo( + int activitiesCount, RMWSConsts.ActivitiesGroupBy groupBy) + throws InterruptedException { + recordCount.set(activitiesCount); + while (recordCount.get() > 0) { + Thread.sleep(1); + } + Iterator>> ite = + lastNActivities.iterator(); + List outList = new ArrayList<>(); + while (ite.hasNext()) { + Pair> pair = ite.next(); + outList.add(new ActivitiesInfo(pair.getRight(), + pair.getLeft().toString(), groupBy)); + } + // reset with new activities + lastNActivities = new ConcurrentLinkedDeque<>(); + return outList; + } + public void recordNextNodeUpdateActivities(String nodeId) { if (nodeId == null) { - recordNextAvailableNode = true; + recordCount.compareAndSet(0, 1); } else { activeRecordedNodes.add(NodeId.fromString(nodeId)); } @@ -348,7 +375,7 @@ protected void serviceStop() throws Exception { } void startNodeUpdateRecording(NodeId nodeID) { - if (recordNextAvailableNode) { + if (recordCount.get() > 0) { recordNextNodeUpdateActivities(nodeID.toString()); } // Removing from activeRecordedNodes immediately is to ensure that @@ -470,14 +497,17 @@ void finishNodeUpdateRecording(NodeId nodeID, String partition) { allocation.setTimestamp(timestamp); allocation.setPartition(partition); } - if (recordNextAvailableNode) { - recordNextAvailableNode = false; + if (recordCount.get() > 0) { + recordCount.getAndDecrement(); } } if (shouldRecordThisNode(nodeID)) { recordingNodesAllocation.get().remove(nodeID); completedNodeAllocations.put(nodeID, value); + if (recordCount.get() >= 0) { + lastNActivities.add(Pair.of(nodeID, value)); + } } } // disable diagnostic collector diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index f6eb2adf726c6..19aa2015013e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -57,7 +57,8 @@ public JAXBContextResolver() throws Exception { FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, AppTimeoutInfo.class, ResourceInformationsInfo.class, ActivitiesInfo.class, AppActivitiesInfo.class, - QueueAclsInfo.class, QueueAclInfo.class}; + QueueAclsInfo.class, QueueAclInfo.class, + BulkActivitiesInfo.class}; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 30406e547d90b..82ceed37c2d5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -81,6 +81,10 @@ public final class RMWSConsts { /** Path for {@code RMWebServiceProtocol#getActivities}. */ public static final String SCHEDULER_ACTIVITIES = "/scheduler/activities"; + /** Path for {@code RMWebServiceProtocol#getBulkActivities}. */ + public static final String SCHEDULER_BULK_ACTIVITIES = + "/scheduler/bulk-activities"; + /** Path for {@code RMWebServiceProtocol#getAppActivities}. */ public static final String SCHEDULER_APP_ACTIVITIES = "/scheduler/app-activities/{appid}"; @@ -252,6 +256,7 @@ public final class RMWSConsts { public static final String ACTIONS = "actions"; public static final String SUMMARIZE = "summarize"; public static final String NAME = "name"; + public static final String ACTIVITIES_COUNT = "activitiesCount"; private RMWSConsts() { // not called diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index a41208e60734c..f2736e3773c1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; /** @@ -212,6 +213,19 @@ AppsInfo getApps(HttpServletRequest hsr, String stateQuery, ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, String groupBy); + /** + * This method retrieve the last n activities inside scheduler and it is + * reachable by using {@link RMWSConsts#SCHEDULER_BULK_ACTIVITIES}. + * + * @param hsr the servlet request + * @param groupBy the groupBy type by which the activities should be + * aggregated. It is a QueryParam. + * @param activitiesCount number of activities + * @return last n activities + */ + BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException; + /** * This method retrieves all the activities for a specific app for a specific * period of time, and it is reachable by using diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index dfdaba9e59a49..7c4e5df5bc7f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -197,6 +197,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo; @@ -242,6 +243,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { public static final String DEFAULT_END_TIME = "-1"; public static final String DEFAULT_INCLUDE_RESOURCE = "false"; public static final String DEFAULT_SUMMARIZE = "false"; + public static final String DEFAULT_ACTIVITIES_COUNT = "10"; + public static final int MAX_ACTIVITIES_COUNT = 500; + private static final String ERROR_MSG = "Not Capacity Scheduler"; @VisibleForTesting boolean isCentralizedNodeLabelConfiguration = true; @@ -697,76 +701,133 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, public ActivitiesInfo getActivities(@Context HttpServletRequest hsr, @QueryParam(RMWSConsts.NODEID) String nodeId, @QueryParam(RMWSConsts.GROUP_BY) String groupBy) { - initForReadableEndpoints(); - YarnScheduler scheduler = rm.getRMContext().getScheduler(); - if (scheduler instanceof AbstractYarnScheduler) { - String errMessage = ""; + initForReadableEndpoints(); - AbstractYarnScheduler abstractYarnScheduler = - (AbstractYarnScheduler) scheduler; + ActivitiesManager activitiesManager = getActivitiesManager(); + if (null == activitiesManager) { + return new ActivitiesInfo(ERROR_MSG, nodeId); + } - ActivitiesManager activitiesManager = - abstractYarnScheduler.getActivitiesManager(); - if (null == activitiesManager) { - errMessage = "Not Capacity Scheduler"; - return new ActivitiesInfo(errMessage, nodeId); - } + RMWSConsts.ActivitiesGroupBy activitiesGroupBy; + try { + activitiesGroupBy = parseActivitiesGroupBy(groupBy); + } catch (IllegalArgumentException e) { + return new ActivitiesInfo(e.getMessage(), nodeId); + } - RMWSConsts.ActivitiesGroupBy activitiesGroupBy; - try { - activitiesGroupBy = parseActivitiesGroupBy(groupBy); - } catch (IllegalArgumentException e) { - return new ActivitiesInfo(e.getMessage(), nodeId); - } + AbstractYarnScheduler abstractYarnScheduler = + (AbstractYarnScheduler) rm.getRMContext().getScheduler(); - List nodeList = - abstractYarnScheduler.getNodeTracker().getAllNodes(); + List nodeList = + abstractYarnScheduler.getNodeTracker().getAllNodes(); - boolean illegalInput = false; + boolean illegalInput = false; + String errMessage = ""; - if (nodeList.size() == 0) { - illegalInput = true; - errMessage = "No node manager running in the cluster"; - } else { - if (nodeId != null) { - String hostName = nodeId; - String portName = ""; - if (nodeId.contains(":")) { - int index = nodeId.indexOf(":"); - hostName = nodeId.substring(0, index); - portName = nodeId.substring(index + 1); - } + if (nodeList.size() == 0) { + illegalInput = true; + errMessage = "No node manager running in the cluster"; + } else { + if (nodeId != null) { + String hostName = nodeId; + String portName = ""; + if (nodeId.contains(":")) { + int index = nodeId.indexOf(":"); + hostName = nodeId.substring(0, index); + portName = nodeId.substring(index + 1); + } - boolean correctNodeId = false; - for (FiCaSchedulerNode node : nodeList) { - if ((portName.equals("") - && node.getRMNode().getHostName().equals(hostName)) - || (!portName.equals("") - && node.getRMNode().getHostName().equals(hostName) - && String.valueOf(node.getRMNode().getCommandPort()) - .equals(portName))) { - correctNodeId = true; - nodeId = node.getNodeID().toString(); - break; - } - } - if (!correctNodeId) { - illegalInput = true; - errMessage = "Cannot find node manager with given node id"; + boolean correctNodeId = false; + for (FiCaSchedulerNode node : nodeList) { + if ((portName.equals("") + && node.getRMNode().getHostName().equals(hostName)) + || (!portName.equals("") + && node.getRMNode().getHostName().equals(hostName) + && String.valueOf(node.getRMNode().getCommandPort()) + .equals(portName))) { + correctNodeId = true; + nodeId = node.getNodeID().toString(); + break; } } + if (!correctNodeId) { + illegalInput = true; + errMessage = "Cannot find node manager with given node id"; + } } + } - if (!illegalInput) { - activitiesManager.recordNextNodeUpdateActivities(nodeId); - return activitiesManager.getActivitiesInfo(nodeId, activitiesGroupBy); - } + if (!illegalInput) { + activitiesManager.recordNextNodeUpdateActivities(nodeId); + return activitiesManager.getActivitiesInfo(nodeId, activitiesGroupBy); + } + + // Return a activities info with error message + return new ActivitiesInfo(errMessage, nodeId); + } + + + @GET + @Path(RMWSConsts.SCHEDULER_BULK_ACTIVITIES) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public BulkActivitiesInfo getBulkActivities( + @Context HttpServletRequest hsr, + @QueryParam(RMWSConsts.GROUP_BY) String groupBy, + @QueryParam(RMWSConsts.ACTIVITIES_COUNT) + @DefaultValue(DEFAULT_ACTIVITIES_COUNT) int activitiesCount) + throws InterruptedException { + + initForReadableEndpoints(); + + ActivitiesManager activitiesManager = getActivitiesManager(); + if (null == activitiesManager) { + throw new BadRequestException(ERROR_MSG); + } - // Return a activities info with error message - return new ActivitiesInfo(errMessage, nodeId); + RMWSConsts.ActivitiesGroupBy activitiesGroupBy; + try { + activitiesGroupBy = parseActivitiesGroupBy(groupBy); + } catch (IllegalArgumentException e) { + throw new BadRequestException(e.getMessage()); } + AbstractYarnScheduler abstractYarnScheduler = + (AbstractYarnScheduler) rm.getRMContext().getScheduler(); + + List nodeList = + abstractYarnScheduler.getNodeTracker().getAllNodes(); + if (nodeList.size() == 0) { + throw new BadRequestException( + "No node manager running in the cluster"); + } + + if (activitiesCount <= 0) { + activitiesCount = Integer.parseInt(DEFAULT_ACTIVITIES_COUNT); + } + activitiesCount = Math.min(activitiesCount, MAX_ACTIVITIES_COUNT); + + List activitiesList = activitiesManager + .recordAndGetBulkActivitiesInfo(activitiesCount, + activitiesGroupBy); + BulkActivitiesInfo bulkActivitiesInfo = new + BulkActivitiesInfo(); + bulkActivitiesInfo.addAll(activitiesList); + + return bulkActivitiesInfo; + } + + private ActivitiesManager getActivitiesManager() { + YarnScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof AbstractYarnScheduler) { + AbstractYarnScheduler abstractYarnScheduler = + (AbstractYarnScheduler) scheduler; + ActivitiesManager activitiesManager = + abstractYarnScheduler.getActivitiesManager(); + return activitiesManager; + } return null; } @@ -788,105 +849,95 @@ public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr, boolean summarize) { initForReadableEndpoints(); - YarnScheduler scheduler = rm.getRMContext().getScheduler(); - if (scheduler instanceof AbstractYarnScheduler) { - AbstractYarnScheduler abstractYarnScheduler = - (AbstractYarnScheduler) scheduler; + ActivitiesManager activitiesManager = getActivitiesManager(); + if (null == activitiesManager) { + return new AppActivitiesInfo(ERROR_MSG, appId); + } - ActivitiesManager activitiesManager = - abstractYarnScheduler.getActivitiesManager(); - if (null == activitiesManager) { - String errMessage = "Not Capacity Scheduler"; - return new AppActivitiesInfo(errMessage, appId); - } + if (appId == null) { + String errMessage = "Must provide an application Id"; + return new AppActivitiesInfo(errMessage, null); + } - if (appId == null) { - String errMessage = "Must provide an application Id"; - return new AppActivitiesInfo(errMessage, null); - } + RMWSConsts.ActivitiesGroupBy activitiesGroupBy; + try { + activitiesGroupBy = parseActivitiesGroupBy(groupBy); + } catch (IllegalArgumentException e) { + return new AppActivitiesInfo(e.getMessage(), appId); + } - RMWSConsts.ActivitiesGroupBy activitiesGroupBy; - try { - activitiesGroupBy = parseActivitiesGroupBy(groupBy); - } catch (IllegalArgumentException e) { - return new AppActivitiesInfo(e.getMessage(), appId); - } + Set requiredActions; + try { + requiredActions = + parseAppActivitiesRequiredActions(getFlatSet(actions)); + } catch (IllegalArgumentException e) { + return new AppActivitiesInfo(e.getMessage(), appId); + } - Set requiredActions; - try { - requiredActions = - parseAppActivitiesRequiredActions(getFlatSet(actions)); - } catch (IllegalArgumentException e) { - return new AppActivitiesInfo(e.getMessage(), appId); - } + Set parsedRequestPriorities; + try { + parsedRequestPriorities = getFlatSet(requestPriorities).stream() + .map(e -> Integer.valueOf(e)).collect(Collectors.toSet()); + } catch (NumberFormatException e) { + return new AppActivitiesInfo("request priorities must be integers!", + appId); + } + Set parsedAllocationRequestIds; + try { + parsedAllocationRequestIds = getFlatSet(allocationRequestIds).stream() + .map(e -> Long.valueOf(e)).collect(Collectors.toSet()); + } catch (NumberFormatException e) { + return new AppActivitiesInfo( + "allocation request Ids must be integers!", appId); + } - Set parsedRequestPriorities; + int limitNum = -1; + if (limit != null) { try { - parsedRequestPriorities = getFlatSet(requestPriorities).stream() - .map(e -> Integer.valueOf(e)).collect(Collectors.toSet()); - } catch (NumberFormatException e) { - return new AppActivitiesInfo("request priorities must be integers!", - appId); - } - Set parsedAllocationRequestIds; - try { - parsedAllocationRequestIds = getFlatSet(allocationRequestIds).stream() - .map(e -> Long.valueOf(e)).collect(Collectors.toSet()); - } catch (NumberFormatException e) { - return new AppActivitiesInfo( - "allocation request Ids must be integers!", appId); - } - - int limitNum = -1; - if (limit != null) { - try { - limitNum = Integer.parseInt(limit); - if (limitNum <= 0) { - return new AppActivitiesInfo( - "limit must be greater than 0!", appId); - } - } catch (NumberFormatException e) { - return new AppActivitiesInfo("limit must be integer!", appId); + limitNum = Integer.parseInt(limit); + if (limitNum <= 0) { + return new AppActivitiesInfo( + "limit must be greater than 0!", appId); } + } catch (NumberFormatException e) { + return new AppActivitiesInfo("limit must be integer!", appId); } + } - double maxTime = 3.0; + double maxTime = 3.0; - if (time != null) { - if (time.contains(".")) { - maxTime = Double.parseDouble(time); - } else { - maxTime = Double.parseDouble(time + ".0"); - } + if (time != null) { + if (time.contains(".")) { + maxTime = Double.parseDouble(time); + } else { + maxTime = Double.parseDouble(time + ".0"); } + } - ApplicationId applicationId; - try { - applicationId = ApplicationId.fromString(appId); - if (requiredActions - .contains(RMWSConsts.AppActivitiesRequiredAction.REFRESH)) { - activitiesManager - .turnOnAppActivitiesRecording(applicationId, maxTime); - } - if (requiredActions - .contains(RMWSConsts.AppActivitiesRequiredAction.GET)) { - AppActivitiesInfo appActivitiesInfo = activitiesManager - .getAppActivitiesInfo(applicationId, parsedRequestPriorities, - parsedAllocationRequestIds, activitiesGroupBy, limitNum, - summarize, maxTime); - return appActivitiesInfo; - } - return new AppActivitiesInfo("Successfully received " - + (actions.size() == 1 ? "action: " : "actions: ") - + StringUtils.join(',', actions), appId); - } catch (Exception e) { - String errMessage = "Cannot find application with given appId"; - LOG.error(errMessage, e); - return new AppActivitiesInfo(errMessage, appId); + ApplicationId applicationId; + try { + applicationId = ApplicationId.fromString(appId); + if (requiredActions + .contains(RMWSConsts.AppActivitiesRequiredAction.REFRESH)) { + activitiesManager + .turnOnAppActivitiesRecording(applicationId, maxTime); } - + if (requiredActions + .contains(RMWSConsts.AppActivitiesRequiredAction.GET)) { + AppActivitiesInfo appActivitiesInfo = activitiesManager + .getAppActivitiesInfo(applicationId, parsedRequestPriorities, + parsedAllocationRequestIds, activitiesGroupBy, limitNum, + summarize, maxTime); + return appActivitiesInfo; + } + return new AppActivitiesInfo("Successfully received " + + (actions.size() == 1 ? "action: " : "actions: ") + + StringUtils.join(',', actions), appId); + } catch (Exception e) { + String errMessage = "Cannot find application with given appId"; + LOG.error(errMessage, e); + return new AppActivitiesInfo(errMessage, appId); } - return null; } private Set getFlatSet(Set set) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/BulkActivitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/BulkActivitiesInfo.java new file mode 100644 index 0000000000000..ad360cc6fc289 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/BulkActivitiesInfo.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; +import java.util.ArrayList; + +/** + * DAO object to display allocation activities. + */ +@XmlRootElement(name = "bulkActivities") +@XmlAccessorType(XmlAccessType.FIELD) +public class BulkActivitiesInfo { + + private ArrayList activities = new ArrayList<>(); + + public BulkActivitiesInfo() { + // JAXB needs this + } + + public void add(ActivitiesInfo activitiesInfo) { + activities.add(activitiesInfo); + } + + public ArrayList getActivities() { + return activities; + } + + public void addAll(List activitiesInfoList) { + activities.addAll(activitiesInfoList); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java index 3c6db7d470620..dce1b64f82e88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java @@ -99,6 +99,8 @@ public final class ActivitiesTestUtils { public static final String FN_SCHEDULER_ACT_ROOT = "activities"; + public static final String FN_SCHEDULER_BULK_ACT_ROOT = "bulkActivities"; + private ActivitiesTestUtils(){} public static List findInAllocations(JSONObject allocationObj, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 1dd80204829ee..f864794d59266 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -21,6 +21,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.core.util.MultivaluedMapImpl; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; @@ -72,6 +73,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_ACT_NAME; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_ACT_ALLOCATIONS_ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_ACT_ROOT; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_BULK_ACT_ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.TOTAL_RESOURCE_INSUFFICIENT_DIAGNOSTIC_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.UNMATCHED_PARTITION_OR_PC_DIAGNOSTIC_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.getFirstSubNodeFromJson; @@ -1586,4 +1588,102 @@ public void testQueueSkippedBecauseOfHeadroom() throws Exception { rm.stop(); } } + + @Test(timeout=30000) + public void testSchedulerBulkActivities() throws Exception { + rm.start(); + + MockNM nm1 = new MockNM("127.0.0.1:1234", 4 * 1024, + rm.getResourceTrackerService()); + MockNM nm2 = new MockNM("127.0.0.2:1234", 4 * 1024, + rm.getResourceTrackerService()); + + nm1.registerNode(); + nm2.registerNode(); + + MockNM[] nms = new MockNM[] {nm1, nm2}; + + try { + + // Validate if response has 5 node activities + int expectedCount = 5; + RESTClient restClient = new RESTClient(5); + restClient.start(); + + sendHeartbeat(restClient, nms); + + JSONObject activitiesJson = restClient.getOutput().getJSONObject( + FN_SCHEDULER_BULK_ACT_ROOT); + Object activities = activitiesJson.get(FN_SCHEDULER_ACT_ROOT); + assertEquals("Number of activities is wrong", expectedCount, + ((JSONArray) activities).length()); + + + // Validate if response does not exceed max 500 + expectedCount = 1000; + restClient = new RESTClient(expectedCount); + restClient.start(); + + sendHeartbeat(restClient, nms); + + activitiesJson = restClient.getOutput().getJSONObject( + FN_SCHEDULER_BULK_ACT_ROOT); + activities = activitiesJson.get(FN_SCHEDULER_ACT_ROOT); + assertEquals("Max Activities Limit does not work", + RMWebServices.MAX_ACTIVITIES_COUNT, + ((JSONArray) activities).length()); + + } finally { + rm.stop(); + } + } + + private class RESTClient extends Thread { + + private int expectedCount; + private boolean done = false; + private JSONObject json; + + RESTClient(int expectedCount) { + this.expectedCount = expectedCount; + } + + boolean isDone() { + return done; + } + + JSONObject getOutput() { + return json; + } + + @Override + public void run() { + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add(RMWSConsts.ACTIVITIES_COUNT, expectedCount); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path(RMWSConsts.SCHEDULER_BULK_ACTIVITIES).queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + + JettyUtils.UTF_8, response.getType().toString()); + json = response.getEntity(JSONObject.class); + done = true; + } + } + + private void sendHeartbeat(RESTClient restClient, MockNM[] nms) + throws Exception { + GenericTestUtils.waitFor(() -> { + try { + for (MockNM nm : nms) { + nm.nodeHeartbeat(true); + } + } catch (Exception e) { + return false; + } + return restClient.isDone(); + }, 10, 20000); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 90ca9928331d6..00a8beb66842f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -199,6 +200,15 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, null, getConf()); } + @Override + public BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + BulkActivitiesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_BULK_ACTIVITIES, + null, null, getConf()); + } + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index b14da6c22c343..ab97b1a7fd9db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.RouterMetrics; import org.apache.hadoop.yarn.server.router.RouterServerUtil; @@ -1147,6 +1148,12 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, throw new NotImplementedException("Code is not implemented"); } + @Override + public BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException { + throw new NotImplementedException("Code is not implemented"); + } + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index 4c694fb4c85bc..bde46484d6b5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -95,6 +96,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices.DEFAULT_ACTIVITIES_COUNT; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices.DEFAULT_SUMMARIZE; /** @@ -457,6 +459,23 @@ public ActivitiesInfo getActivities(@Context HttpServletRequest hsr, .getActivities(hsr, nodeId, groupBy); } + @GET + @Path(RMWSConsts.SCHEDULER_BULK_ACTIVITIES) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public BulkActivitiesInfo getBulkActivities( + @Context HttpServletRequest hsr, + @QueryParam(RMWSConsts.GROUP_BY) String groupBy, + @QueryParam(RMWSConsts.ACTIVITIES_COUNT) + @DefaultValue(DEFAULT_ACTIVITIES_COUNT) int activitiesCount) + throws InterruptedException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor().getBulkActivities(hsr, groupBy, + activitiesCount); + } + @GET @Path(RMWSConsts.SCHEDULER_APP_ACTIVITIES) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index b626a8a21de43..05a088df781be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.router.webapp.RouterWebServices.RequestInterceptorChainWrapper; @@ -179,6 +180,12 @@ protected ActivitiesInfo getActivities(String user) createHttpServletRequest(user), null, null); } + protected BulkActivitiesInfo getBulkActivities(String user) + throws InterruptedException { + return routerWebService.getBulkActivities( + createHttpServletRequest(user), null, 0); + } + protected AppActivitiesInfo getAppActivities(String user) throws IOException, InterruptedException { return routerWebService.getAppActivities(createHttpServletRequest(user), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index f6dbb7ff388a8..67c9d671fb159 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -138,6 +139,12 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, return new ActivitiesInfo(); } + @Override + public BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException{ + return new BulkActivitiesInfo(); + } + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 55de7a4a5a9f8..142a6511b93a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -166,6 +167,13 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, return getNextInterceptor().getActivities(hsr, nodeId, groupBy); } + @Override + public BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException { + return getNextInterceptor().getBulkActivities(hsr, groupBy, + activitiesCount); + } + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index a30221d7a6ce3..879075e794446 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -5746,6 +5746,140 @@ Response Body: ``` +Scheduler Bulk Activities API +-------------------------------- + + The scheduler bulk activities RESTful API can fetch scheduler activities info recorded for multiple scheduling cycle. This may take time +to return as it internally waits until a certain amount of records are generated specified by activitiesCount. + +### URI + + * http://rm-http-address:port/ws/v1/cluster/scheduler/bulk-activities + +### HTTP Operations Supported + + * GET + +### Query Parameters Supported + +Multiple parameters can be specified for GET operations. + + * activitiesCount - number of schecduling cycle to record with maximum of 500. + * groupBy - aggregation type of application activities, currently only support "diagnostic" with which + user can query aggregated activities grouped by allocation state and diagnostic. + + +### Response Examples + +**JSON response** + +HTTP Request: + + Accept: application/json + GET http://rm-http-address:port/ws/v1/cluster/scheduler/bulk-activities?activitiesCount=2 + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) + +Response Body: + +Following is an output example with query parameter activitiesCount set to 2. This fetches scheduler activities info +recorded in last two scheduling cycle. + +```json +{ + "bulkActivities": { + "activities": [ + { + "nodeId": "127.0.0.1:1234", + "timestamp": 1593684431432, + "dateTime": "Thu Jul 02 10:07:11 UTC 2020", + "allocations": [ + { + "partition": "", + "finalAllocationState": "SKIPPED", + "root": { + "name": "root", + "allocationState": "SKIPPED", + "diagnostic": "Queue does not need more resource" + } + } + ] + }, + { + "nodeId": "127.0.0.2:1234", + "timestamp": 1593684431432, + "dateTime": "Thu Jul 02 10:07:11 UTC 2020", + "allocations": [ + { + "partition": "", + "finalAllocationState": "SKIPPED", + "root": { + "name": "root", + "allocationState": "SKIPPED", + "diagnostic": "Queue does not need more resource" + } + } + ] + } + ] + } +} +``` + +**XML response** + +HTTP Request: + + Accept: application/xml + GET http://rm-http-address:port/ws/v1/cluster/scheduler/bulk-activities?activitiesCount=2 + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml; charset=utf-8 + Transfer-Encoding: chunked + +Response Body: + +```xml + + + 127.0.0.1:1234 + 1593683816380 + Thu Jul 02 09:56:56 UTC 2020 + + + SKIPPED + + root + SKIPPED + Queue does not need more resource + + + + + 127.0.0.2:1234 + 1593683816385 + Thu Jul 02 09:56:56 UTC 2020 + + + SKIPPED + + root + SKIPPED + Queue does not need more resource + + + + +``` + + Application Activities API -------------------------------- From e60096c377d8a3cb5bed3992352779195be95bb4 Mon Sep 17 00:00:00 2001 From: belugabehr <12578579+belugabehr@users.noreply.github.com> Date: Fri, 24 Jul 2020 05:37:28 -0400 Subject: [PATCH 013/335] HADOOP-17141. Add Capability To Get Text Length (#2157) Contributed by David Mollitor --- .../main/java/org/apache/hadoop/io/Text.java | 18 ++++++++++++++++++ .../java/org/apache/hadoop/io/TestText.java | 10 ++++++++++ 2 files changed, 28 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java index 716de3deb4278..6022b99544114 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java @@ -77,6 +77,7 @@ protected CharsetDecoder initialValue() { private byte[] bytes = EMPTY_BYTES; private int length = 0; + private int textLength = -1; /** * Construct an empty text string. @@ -131,6 +132,17 @@ public int getLength() { return length; } + /** + * Returns the length of this text. The length is equal to the number of + * Unicode code units in the text. + */ + public int getTextLength() { + if (textLength < 0) { + textLength = toString().length(); + } + return textLength; + } + /** * Returns the Unicode Scalar Value (32-bit integer value) * for the character at position. Note that this @@ -204,6 +216,7 @@ public void set(String string) { ByteBuffer bb = encode(string, true); bytes = bb.array(); length = bb.limit(); + textLength = string.length(); } catch (CharacterCodingException e) { throw new RuntimeException("Should not have happened", e); } @@ -221,6 +234,7 @@ public void set(byte[] utf8) { */ public void set(Text other) { set(other.getBytes(), 0, other.getLength()); + this.textLength = other.textLength; } /** @@ -234,6 +248,7 @@ public void set(byte[] utf8, int start, int len) { ensureCapacity(len); System.arraycopy(utf8, start, bytes, 0, len); this.length = len; + this.textLength = -1; } /** @@ -251,6 +266,7 @@ public void append(byte[] utf8, int start, int len) { } System.arraycopy(utf8, start, bytes, length, len); length += len; + textLength = -1; } /** @@ -263,6 +279,7 @@ public void append(byte[] utf8, int start, int len) { */ public void clear() { length = 0; + textLength = -1; } /** @@ -327,6 +344,7 @@ public void readWithKnownLength(DataInput in, int len) throws IOException { ensureCapacity(len); in.readFully(bytes, 0, len); length = len; + textLength = -1; } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java index 54df39955d6cf..700e106271ea4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java @@ -268,6 +268,8 @@ public void testClear() throws Exception { 0, text.getBytes().length); assertEquals("String's length must be zero", 0, text.getLength()); + assertEquals("String's text length must be zero", + 0, text.getTextLength()); // Test if clear works as intended text = new Text("abcd\u20acbdcd\u20ac"); @@ -280,6 +282,8 @@ public void testClear() throws Exception { text.getBytes().length >= len); assertEquals("Length of the string must be reset to 0 after clear()", 0, text.getLength()); + assertEquals("Text length of the string must be reset to 0 after clear()", + 0, text.getTextLength()); } @Test @@ -288,9 +292,12 @@ public void testTextText() throws CharacterCodingException { Text b=new Text("a"); b.set(a); assertEquals("abc", b.toString()); + assertEquals(3, a.getTextLength()); + assertEquals(3, b.getTextLength()); a.append("xdefgxxx".getBytes(), 1, 4); assertEquals("modified aliased string", "abc", b.toString()); assertEquals("appended string incorrectly", "abcdefg", a.toString()); + assertEquals("This should reflect in the lenght", 7, a.getTextLength()); // add an extra byte so that capacity = 10 and length = 8 a.append(new byte[]{'d'}, 0, 1); assertEquals(10, a.getBytes().length); @@ -392,16 +399,19 @@ public void testReadWithKnownLength() throws IOException { in.reset(inputBytes, inputBytes.length); text.readWithKnownLength(in, 5); assertEquals("hello", text.toString()); + assertEquals(5, text.getTextLength()); // Read longer length, make sure it lengthens in.reset(inputBytes, inputBytes.length); text.readWithKnownLength(in, 7); assertEquals("hello w", text.toString()); + assertEquals(7, text.getTextLength()); // Read shorter length, make sure it shortens in.reset(inputBytes, inputBytes.length); text.readWithKnownLength(in, 2); assertEquals("he", text.toString()); + assertEquals(2, text.getTextLength()); } /** From ac5f21dbef0f0ad4210e4027f53877760fa606a5 Mon Sep 17 00:00:00 2001 From: Eric Badger Date: Fri, 24 Jul 2020 22:35:16 +0000 Subject: [PATCH 014/335] YARN-4771. Some containers can be skipped during log aggregation after NM restart. Contributed by Jason Lowe and Jim Brennan. --- .../nodemanager/NodeStatusUpdaterImpl.java | 11 +++++---- .../nodemanager/TestNodeStatusUpdater.java | 24 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 0725d423096c7..37da31a322b3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -777,8 +777,13 @@ public void removeVeryOldStoppedContainersFromCache() { while (i.hasNext()) { Entry mapEntry = i.next(); ContainerId cid = mapEntry.getKey(); - if (mapEntry.getValue() < currentTime) { - if (!context.getContainers().containsKey(cid)) { + if (mapEntry.getValue() >= currentTime) { + break; + } + if (!context.getContainers().containsKey(cid)) { + ApplicationId appId = + cid.getApplicationAttemptId().getApplicationId(); + if (isApplicationStopped(appId)) { i.remove(); try { context.getNMStateStore().removeContainer(cid); @@ -786,8 +791,6 @@ public void removeVeryOldStoppedContainersFromCache() { LOG.error("Unable to remove container " + cid + " in store", e); } } - } else { - break; } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index c0831ee022dc9..2477af2512392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -931,9 +931,8 @@ public void deleteBaseDir() throws IOException { public void testRecentlyFinishedContainers() throws Exception { NodeManager nm = new NodeManager(); YarnConfiguration conf = new YarnConfiguration(); - conf.set( - NodeStatusUpdaterImpl.YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS, - "10000"); + conf.setInt(NodeStatusUpdaterImpl. + YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS, 1); nm.init(conf); NodeStatusUpdaterImpl nodeStatusUpdater = (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater(); @@ -948,18 +947,17 @@ public void testRecentlyFinishedContainers() throws Exception { nodeStatusUpdater.addCompletedContainer(cId); Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId)); + // verify container remains even after expiration if app + // is still active nm.getNMContext().getContainers().remove(cId); - long time1 = System.currentTimeMillis(); - int waitInterval = 15; - while (waitInterval-- > 0 - && nodeStatusUpdater.isContainerRecentlyStopped(cId)) { - nodeStatusUpdater.removeVeryOldStoppedContainersFromCache(); - Thread.sleep(1000); - } - long time2 = System.currentTimeMillis(); - // By this time the container will be removed from cache. need to verify. + Thread.sleep(10); + nodeStatusUpdater.removeVeryOldStoppedContainersFromCache(); + Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId)); + + // complete the application and verify container is removed + nm.getNMContext().getApplications().remove(appId); + nodeStatusUpdater.removeVeryOldStoppedContainersFromCache(); Assert.assertFalse(nodeStatusUpdater.isContainerRecentlyStopped(cId)); - Assert.assertTrue((time2 - time1) >= 10000 && (time2 - time1) <= 250000); } @Test(timeout = 90000) From 4b1816c7d0188363896505d0f0f93cb58d44bcd9 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Sun, 26 Jul 2020 06:38:54 +0900 Subject: [PATCH 015/335] HADOOP-17153. Add boost installation steps to build instruction on CentOS 8. (#2169) --- BUILDING.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/BUILDING.txt b/BUILDING.txt index c96c851204e75..78843ccfb95b4 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -449,6 +449,14 @@ Building on CentOS 8 * Install libraries provided by CentOS 8. $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel +* Install boost. + $ curl -L -o boost_1_72_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download + $ tar xjf boost_1_72_0.tar.bz2 + $ cd boost_1_72_0 + $ ./bootstrap.sh --prefix=/usr/local + $ ./b2 + $ sudo ./b2 install + * Install optional dependencies (snappy-devel). $ sudo dnf --enablerepo=PowerTools snappy-devel From e277d338da2675262b4d6985e08c78e408b927a3 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 27 Jul 2020 00:51:44 +0900 Subject: [PATCH 016/335] YARN-10367. Failed to get nodejs 10.21.0 when building docker image (#2171) --- dev-support/docker/Dockerfile | 5 +++-- dev-support/docker/Dockerfile_aarch64 | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index f72fa4659009a..2af73eb8b75b0 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -138,10 +138,11 @@ RUN pip2 install \ RUN pip2 install python-dateutil==2.7.3 ### -# Install node.js 10.21.0 for web UI framework (4.2.6 ships with Xenial) +# Install node.js 10.x for web UI framework (4.2.6 ships with Xenial) ### +# hadolint ignore=DL3008 RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \ - && apt-get install -y --no-install-recommends nodejs=10.21.0-1nodesource1 \ + && apt-get install -y --no-install-recommends nodejs \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && npm install -g bower@1.8.8 diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 5fd646fb9c08a..2d43bb8bfaac4 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -141,10 +141,11 @@ RUN pip2 install \ RUN pip2 install python-dateutil==2.7.3 ### -# Install node.js 10.21.0 for web UI framework (4.2.6 ships with Xenial) +# Install node.js 10.x for web UI framework (4.2.6 ships with Xenial) ### +# hadolint ignore=DL3008 RUN curl -L -s -S https://deb.nodesource.com/setup_10.x | bash - \ - && apt-get install -y --no-install-recommends nodejs=10.21.0-1nodesource1 \ + && apt-get install -y --no-install-recommends nodejs \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && npm install -g bower@1.8.8 From d02be17a269cb7f5ac51910802683a1729510250 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 27 Jul 2020 01:55:04 +0900 Subject: [PATCH 017/335] YARN-10362. Javadoc for TimelineReaderAuthenticationFilterInitializer is broken. Contributed by Xieming Li. --- .../TimelineReaderAuthenticationFilterInitializer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java index 6a3658d2da8ef..bf04ffc2818c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java @@ -36,7 +36,8 @@ public class TimelineReaderAuthenticationFilterInitializer extends *

* Propagates to {@link AuthenticationFilter} configuration all * YARN configuration properties prefixed with - * {@value TimelineAuthenticationFilterInitializer#PREFIX}. + * {@value + * org.apache.hadoop.yarn.conf.YarnConfiguration#TIMELINE_HTTP_AUTH_PREFIX}. * * @param container * The filter container From 60a254621a3d07a6c3c4611ea86cffd9625e8fb2 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 24 Jul 2020 16:59:15 +0530 Subject: [PATCH 018/335] YARN-10366. Fix Yarn rmadmin help message shows two labels for one node for --replaceLabelsOnNode. Contributed by Tanu Ajmera. --- .../java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java | 4 ++-- .../org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 489509b849d1c..69fd1178ded23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -140,7 +140,7 @@ public class RMAdminCLI extends HAAdmin { "remove from cluster node labels")) .put("-replaceLabelsOnNode", new UsageInfo( - "<\"node1[:port]=label1,label2 node2[:port]=label1,label2\"> " + "<\"node1[:port]=label1 node2[:port]=label2\"> " + "[-failOnUnknownNodes] ", "replace labels on nodes" + " (please note that we do not support specifying multiple" @@ -280,7 +280,7 @@ private static void printHelp(String cmd, boolean isHAEnabled) { + "label2(exclusive=false),label3\">]" + " [-removeFromClusterNodeLabels ]" + " [-replaceLabelsOnNode " - + "<\"node1[:port]=label1,label2 node2[:port]=label1\"> " + + "<\"node1[:port]=label1 node2[:port]=label2\"> " + "[-failOnUnknownNodes]]" + " [-directlyAccessNodeLabelStore]" + " [-refreshClusterMaxPriority]" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 40b393ff3f38d..e0b0041afac7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -703,7 +703,7 @@ public void testHelp() throws Exception { "<\"label1(exclusive=true),label2(exclusive=false),label3\">] " + "[-removeFromClusterNodeLabels ] " + "[-replaceLabelsOnNode " + - "<\"node1[:port]=label1,label2 node2[:port]=label1\"> " + + "<\"node1[:port]=label1 node2[:port]=label2\"> " + "[-failOnUnknownNodes]] " + "[-directlyAccessNodeLabelStore] [-refreshClusterMaxPriority] " + "[-updateNodeResource [NodeID] [MemSize] [vCores] " @@ -795,7 +795,7 @@ public void testHelp() throws Exception { + " [username]] [-addToClusterNodeLabels <\"label1(exclusive=true)," + "label2(exclusive=false),label3\">]" + " [-removeFromClusterNodeLabels ] [-replaceLabelsOnNode " - + "<\"node1[:port]=label1,label2 node2[:port]=label1\"> " + + "<\"node1[:port]=label1 node2[:port]=label2\"> " + "[-failOnUnknownNodes]] [-directlyAccessNodeLabelStore] " + "[-refreshClusterMaxPriority] " + "[-updateNodeResource [NodeID] [MemSize] [vCores] " From 026dce5334bca3b0aa9b05a6debe72db1e01842e Mon Sep 17 00:00:00 2001 From: touchida <56789230+touchida@users.noreply.github.com> Date: Tue, 28 Jul 2020 01:55:11 +0900 Subject: [PATCH 019/335] HDFS-15465. Support WebHDFS accesses to the data stored in secure Datanode through insecure Namenode. (#2135) --- .../web/webhdfs/DataNodeUGIProvider.java | 10 ++++-- .../datanode/web/webhdfs/ParameterParser.java | 3 ++ .../datanode/web/webhdfs/WebHdfsHandler.java | 4 +-- .../web/webhdfs/TestDataNodeUGIProvider.java | 32 +++++++++++++++++++ .../web/webhdfs/TestParameterParser.java | 9 ++++++ 5 files changed, 53 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java index 366f47f29631d..d7e5f9f155130 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java @@ -72,9 +72,12 @@ UserGroupInformation ugi() throws IOException { UserGroupInformation ugi; try { - if (UserGroupInformation.isSecurityEnabled()) { - final Token token = params.delegationToken(); + final Token token = params.delegationToken(); + // Create nonTokenUGI when token is null regardless of security. + // This makes it possible to access the data stored in secure DataNode + // through insecure Namenode. + if (UserGroupInformation.isSecurityEnabled() && token != null) { ugi = ugiCache.get(buildTokenCacheKey(token), new Callable() { @Override @@ -134,7 +137,8 @@ private String buildNonTokenCacheKey(String doAsUserFromQuery, return key; } - private UserGroupInformation nonTokenUGI(String usernameFromQuery, + @VisibleForTesting + UserGroupInformation nonTokenUGI(String usernameFromQuery, String doAsUserFromQuery, String remoteUser) throws IOException { UserGroupInformation ugi = UserGroupInformation diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java index 2b3a39358de6e..c4588fc65744e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java @@ -123,6 +123,9 @@ boolean noredirect() { Token delegationToken() throws IOException { String delegation = param(DelegationParam.NAME); + if (delegation == null) { + return null; + } final Token token = new Token(); token.decodeFromUrlString(delegation); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java index ff68a7ee7e4d0..834d9d5e1daee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java @@ -335,8 +335,8 @@ private static void writeContinueHeader(ChannelHandlerContext ctx) { } private void injectToken() throws IOException { - if (UserGroupInformation.isSecurityEnabled()) { - Token token = params.delegationToken(); + Token token = params.delegationToken(); + if (UserGroupInformation.isSecurityEnabled() && token != null) { token.setKind(HDFS_DELEGATION_KIND); ugi.addToken(token); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java index de88c51ed2fdb..102418517f64d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import io.netty.handler.codec.http.QueryStringDecoder; import java.io.IOException; @@ -31,6 +33,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.web.WebHdfsConstants; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; @@ -186,6 +189,35 @@ public void testUGICacheInSecure() throws Exception { ugi11, url22); } + @Test + public void testUGINullTokenSecure() throws IOException { + SecurityUtil.setAuthenticationMethod(KERBEROS, conf); + UserGroupInformation.setConfiguration(conf); + + String uri1 = WebHdfsFileSystem.PATH_PREFIX + + PATH + + "?op=OPEN" + + Param.toSortedString("&", new OffsetParam((long) OFFSET), + new LengthParam((long) LENGTH), new UserParam("root")); + + ParameterParser params = new ParameterParser( + new QueryStringDecoder(URI.create(uri1)), conf); + + DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params); + + String usernameFromQuery = params.userName(); + String doAsUserFromQuery = params.doAsUser(); + String remoteUser = usernameFromQuery == null ? JspHelper + .getDefaultWebUserName(params.conf()) + : usernameFromQuery; + + DataNodeUGIProvider spiedUGIProvider = spy(ugiProvider); + spiedUGIProvider.ugi(); + + verify(spiedUGIProvider).nonTokenUGI(usernameFromQuery, doAsUserFromQuery, + remoteUser); + } + /** * Wait for expiration of entries from the UGI cache. We need to be careful * not to touch the entries in the cache while we're waiting for expiration. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java index 235d051400be4..40409985c3f02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java @@ -55,6 +55,15 @@ public void testDeserializeHAToken() throws IOException { Assert.assertTrue(HAUtilClient.isTokenForLogicalUri(tok2)); } + @Test + public void testNullToken() throws IOException { + Configuration conf = new Configuration(); + QueryStringDecoder decoder = new QueryStringDecoder( + WebHdfsHandler.WEBHDFS_PREFIX + "/test"); + ParameterParser testParser = new ParameterParser(decoder, conf); + Assert.assertNull(testParser.delegationToken()); + } + @Test public void testDecodePath() { final String ESCAPED_PATH = "/test%25+1%26%3Dtest?op=OPEN&foo=bar"; From 5dadf963d3639cc6d37902d9c7beaacdafac0e9c Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Tue, 28 Jul 2020 11:55:47 +0530 Subject: [PATCH 020/335] YARN-10208. Add capacityScheduler metric for NODE_UPDATE interval. Contributed by Pranjal Protim Borah. --- .../scheduler/capacity/CapacityScheduler.java | 14 ++++++++++++++ .../capacity/CapacitySchedulerMetrics.java | 12 ++++++++++++ .../TestCapacitySchedulerMetrics.java | 6 ++++++ 3 files changed, 32 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index bd2acd7611536..699c8310964f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1828,6 +1828,7 @@ public void handle(SchedulerEvent event) { case NODE_UPDATE: { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; + updateSchedulerNodeHBIntervalMetrics(nodeUpdatedEvent); nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; @@ -2114,6 +2115,19 @@ private void removeNode(RMNode nodeInfo) { } } + private void updateSchedulerNodeHBIntervalMetrics( + NodeUpdateSchedulerEvent nodeUpdatedEvent) { + // Add metrics for evaluating the time difference between heartbeats. + SchedulerNode node = + nodeTracker.getNode(nodeUpdatedEvent.getRMNode().getNodeID()); + if (node != null) { + long lastInterval = + Time.monotonicNow() - node.getLastHeartbeatMonotonicTime(); + CapacitySchedulerMetrics.getMetrics() + .addSchedulerNodeHBInterval(lastInterval); + } + } + @Override protected void completedContainerInternal( RMContainer rmContainer, ContainerStatus containerStatus, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerMetrics.java index 5f8988b077811..be30c60cdadd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerMetrics.java @@ -26,6 +26,7 @@ import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.metrics2.lib.MutableRate; import java.util.concurrent.atomic.AtomicBoolean; @@ -49,6 +50,8 @@ public class CapacitySchedulerMetrics { @Metric("Scheduler commit success") MutableRate commitSuccess; @Metric("Scheduler commit failure") MutableRate commitFailure; @Metric("Scheduler node update") MutableRate nodeUpdate; + @Metric("Scheduler node heartbeat interval") MutableQuantiles + schedulerNodeHBInterval; private static volatile CapacitySchedulerMetrics INSTANCE = null; private static MetricsRegistry registry; @@ -116,4 +119,13 @@ public long getNumOfAllocates() { public long getNumOfCommitSuccess() { return this.commitSuccess.lastStat().numSamples(); } + + public void addSchedulerNodeHBInterval(long heartbeatInterval) { + schedulerNodeHBInterval.add(heartbeatInterval); + } + + @VisibleForTesting + public long getNumOfSchedulerNodeHBInterval() { + return this.schedulerNodeHBInterval.getEstimator().getCount(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java index 62690e9e30ed6..99b3983f863eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestCapacitySchedulerMetrics.java @@ -71,6 +71,9 @@ public RMNodeLabelsManager createNodeLabelManager() { try { GenericTestUtils.waitFor(() -> csMetrics.getNumOfNodeUpdate() == 2, 100, 3000); + GenericTestUtils + .waitFor(() -> csMetrics.getNumOfSchedulerNodeHBInterval() == 2, + 100, 3000); } catch(TimeoutException e) { Assert.fail("CS metrics not updated on node-update events."); } @@ -101,6 +104,9 @@ public RMNodeLabelsManager createNodeLabelManager() { // Verify HB metrics updated GenericTestUtils.waitFor(() -> csMetrics.getNumOfNodeUpdate() == 4, 100, 3000); + GenericTestUtils + .waitFor(() -> csMetrics.getNumOfSchedulerNodeHBInterval() == 4, + 100, 3000); // For async mode, the number of alloc might be bigger than 1 Assert.assertTrue(csMetrics.getNumOfAllocates() > 0); // But there will be only 2 successful commit (1 AM + 1 task) From 3eaf62726ffe90b3b096798fe501abd1ed0c5f15 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Tue, 28 Jul 2020 13:43:19 -0700 Subject: [PATCH 021/335] YARN-10343. Legacy RM UI should include labeled metrics for allocated, total, and reserved resources. Contributed by Eric Payne --- .../scheduler/ResourceUsage.java | 4 ++ .../webapp/MetricsOverviewTable.java | 49 ++++++++++++++++--- .../webapp/dao/ClusterMetricsInfo.java | 26 ++++++++++ .../webapp/TestRMWebServices.java | 2 +- 4 files changed, 72 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java index c46c9112ea8fd..18fd6c3567dc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java @@ -200,6 +200,10 @@ public Resource getAllUsed() { return _getAll(ResourceType.USED); } + public Resource getAllReserved() { + return _getAll(ResourceType.RESERVED); + } + // Cache Used public Resource getCachedUsed() { return _get(NL, ResourceType.CACHED_USED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index 806b63640990b..fbaeafd9218ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo; @@ -60,7 +61,38 @@ protected void render(Block html) { ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm); DIV div = html.div().$class("metrics"); - + + long usedMemoryBytes = 0; + long totalMemoryBytes = 0; + long reservedMemoryBytes = 0; + long usedVCores = 0; + long totalVCores = 0; + long reservedVCores = 0; + if (clusterMetrics.getCrossPartitionMetricsAvailable()) { + ResourceInfo usedAllPartitions = + clusterMetrics.getTotalUsedResourcesAcrossPartition(); + ResourceInfo totalAllPartitions = + clusterMetrics.getTotalClusterResourcesAcrossPartition(); + ResourceInfo reservedAllPartitions = + clusterMetrics.getTotalReservedResourcesAcrossPartition(); + usedMemoryBytes = usedAllPartitions.getMemorySize() * BYTES_IN_MB; + totalMemoryBytes = totalAllPartitions.getMemorySize() * BYTES_IN_MB; + reservedMemoryBytes = reservedAllPartitions.getMemorySize() * BYTES_IN_MB; + usedVCores = usedAllPartitions.getvCores(); + totalVCores = totalAllPartitions.getvCores(); + reservedVCores = reservedAllPartitions.getvCores(); + // getTotalUsedResourcesAcrossPartition includes reserved resources. + usedMemoryBytes -= reservedMemoryBytes; + usedVCores -= reservedVCores; + } else { + usedMemoryBytes = clusterMetrics.getAllocatedMB() * BYTES_IN_MB; + totalMemoryBytes = clusterMetrics.getTotalMB() * BYTES_IN_MB; + reservedMemoryBytes = clusterMetrics.getReservedMB() * BYTES_IN_MB; + usedVCores = clusterMetrics.getAllocatedVirtualCores(); + totalVCores = clusterMetrics.getTotalVirtualCores(); + reservedVCores = clusterMetrics.getReservedVirtualCores(); + } + div.h3("Cluster Metrics"). table("#metricsoverview"). thead().$class("ui-widget-header"). @@ -89,13 +121,14 @@ protected void render(Block html) { clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled() ) ). - td(String.valueOf(clusterMetrics.getContainersAllocated())). - td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)). - td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())). - td(String.valueOf(clusterMetrics.getTotalVirtualCores())). - td(String.valueOf(clusterMetrics.getReservedVirtualCores())). + td(String.valueOf( + clusterMetrics.getTotalAllocatedContainersAcrossPartition())). + td(StringUtils.byteDesc(usedMemoryBytes)). + td(StringUtils.byteDesc(totalMemoryBytes)). + td(StringUtils.byteDesc(reservedMemoryBytes)). + td(String.valueOf(usedVCores)). + td(String.valueOf(totalVCores)). + td(String.valueOf(reservedVCores)). __(). __().__(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index d6e4828fbc157..ee76eeacad279 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; @XmlRootElement(name = "clusterMetrics") @XmlAccessorType(XmlAccessType.FIELD) @@ -69,6 +70,14 @@ public class ClusterMetricsInfo { // Total registered resources of the cluster, including all partitions private ResourceInfo totalClusterResourcesAcrossPartition; + // Total reserved resources of the cluster, including all partitions. + private ResourceInfo totalReservedResourcesAcrossPartition; + + // Total allocated containers across all partitions. + private int totalAllocatedContainersAcrossPartition; + + private boolean crossPartitionMetricsAvailable = false; + public ClusterMetricsInfo() { } // JAXB needs this @@ -115,6 +124,11 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { cs.getRootQueue().getQueueResourceUsage().getAllUsed()); totalClusterResourcesAcrossPartition = new ResourceInfo( cs.getClusterResource()); + totalReservedResourcesAcrossPartition = new ResourceInfo( + cs.getRootQueue().getQueueResourceUsage().getAllReserved()); + totalAllocatedContainersAcrossPartition = + ((ParentQueue) cs.getRootQueue()).getNumContainers(); + crossPartitionMetricsAvailable = true; } } else { this.totalMB = availableMB + allocatedMB; @@ -346,4 +360,16 @@ public ResourceInfo getTotalUsedResourcesAcrossPartition() { public ResourceInfo getTotalClusterResourcesAcrossPartition() { return totalClusterResourcesAcrossPartition; } + + public ResourceInfo getTotalReservedResourcesAcrossPartition() { + return totalReservedResourcesAcrossPartition; + } + + public int getTotalAllocatedContainersAcrossPartition() { + return totalAllocatedContainersAcrossPartition; + } + + public boolean getCrossPartitionMetricsAvailable() { + return crossPartitionMetricsAvailable; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index afc37fa57a088..079737cc81531 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -474,7 +474,7 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("incorrect number of elements", 27, clusterinfo.length()); + assertEquals("incorrect number of elements", 29, clusterinfo.length()); verifyClusterMetrics( clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"), clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"), From 68287371ccc66da80e6a3d7981ae6c7ce7238920 Mon Sep 17 00:00:00 2001 From: bshashikant Date: Wed, 29 Jul 2020 21:33:25 +0530 Subject: [PATCH 022/335] HDFS-15488. Add a command to list all snapshots for a snaphottable root with snapshot Ids. (#2166) --- .../dev-support/findbugsExcludeFile.xml | 1 + .../org/apache/hadoop/hdfs/DFSClient.java | 19 ++ .../hadoop/hdfs/DFSOpsCountStatistics.java | 1 + .../hadoop/hdfs/DistributedFileSystem.java | 14 ++ .../hadoop/hdfs/protocol/ClientProtocol.java | 12 + .../hadoop/hdfs/protocol/SnapshotStatus.java | 226 ++++++++++++++++++ .../ClientNamenodeProtocolTranslatorPB.java | 22 ++ .../hdfs/protocolPB/PBHelperClient.java | 77 ++++++ .../main/proto/ClientNamenodeProtocol.proto | 10 + .../src/main/proto/hdfs.proto | 21 ++ .../hadoop/hdfs/protocol/TestReadOnly.java | 1 + .../router/RouterClientProtocol.java | 7 + .../federation/router/RouterRpcServer.java | 7 + .../federation/router/RouterSnapshot.java | 35 +++ .../federation/router/TestRouterRpc.java | 16 +- .../hadoop-hdfs/src/main/bin/hdfs | 4 + .../hadoop-hdfs/src/main/bin/hdfs.cmd | 8 +- ...amenodeProtocolServerSideTranslatorPB.java | 24 ++ .../hdfs/server/namenode/FSDirSnapshotOp.java | 17 ++ .../hdfs/server/namenode/FSNamesystem.java | 34 ++- .../server/namenode/NameNodeRpcServer.java | 11 + .../namenode/metrics/NameNodeMetrics.java | 7 + .../namenode/snapshot/SnapshotManager.java | 39 ++- .../apache/hadoop/hdfs/tools/AdminHelper.java | 2 +- .../hdfs/tools/snapshot/LsSnapshot.java | 63 +++++ .../src/site/markdown/HDFSCommands.md | 10 + .../src/site/markdown/HdfsSnapshots.md | 17 ++ .../namenode/snapshot/TestListSnapshot.java | 134 +++++++++++ 28 files changed, 833 insertions(+), 6 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshot.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml index 278d01dc22d0f..c96b3a99bd1c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -22,6 +22,7 @@ + diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 5a6a0f65f12f6..d781dd9ac45db 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -150,6 +150,7 @@ import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; @@ -2190,6 +2191,24 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() } } + /** + * Get listing of all the snapshots for a snapshottable directory. + * + * @return Information about all the snapshots for a snapshottable directory + * @throws IOException If an I/O error occurred + * @see ClientProtocol#getSnapshotListing(String) + */ + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + checkOpen(); + try (TraceScope ignored = tracer.newScope("getSnapshotListing")) { + return namenode.getSnapshotListing(snapshotRoot); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + /** * Allow snapshot on a directory. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index 2113ae5c63544..fdd0072905fd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -111,6 +111,7 @@ public enum OpType { SET_XATTR("op_set_xattr"), GET_SNAPSHOT_DIFF("op_get_snapshot_diff"), GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"), + GET_SNAPSHOT_LIST("op_get_snapshot_list"), TRUNCATE(CommonStatisticNames.OP_TRUNCATE), UNSET_EC_POLICY("op_unset_ec_policy"), UNSET_STORAGE_POLICY("op_unset_storage_policy"); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 450862b777078..37d0226a3a326 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -109,6 +109,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry; import org.apache.hadoop.hdfs.client.impl.SnapshotDiffReportGenerator; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; @@ -2148,6 +2149,19 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return dfs.getSnapshottableDirListing(); } + /** + * @return all the snapshots for a snapshottable directory + * @throws IOException + */ + public SnapshotStatus[] getSnapshotListing(Path snapshotRoot) + throws IOException { + Path absF = fixRelativePart(snapshotRoot); + statistics.incrementReadOps(1); + storageStatistics + .incrementOpCounter(OpType.GET_SNAPSHOT_LIST); + return dfs.getSnapshotListing(getPathName(absF)); + } + @Override public void deleteSnapshot(final Path snapshotDir, final String snapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 2f4dfb9b46cc1..ea90645ca082b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -727,6 +727,18 @@ BatchedDirectoryListing getBatchedListing( SnapshottableDirectoryStatus[] getSnapshottableDirListing() throws IOException; + /** + * Get listing of all the snapshots for a snapshottable directory. + * + * @return Information about all the snapshots for a snapshottable directory + * @throws IOException If an I/O error occurred + */ + @Idempotent + @ReadOnly(isCoordinated = true) + SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException; + + /////////////////////////////////////// // System issues and management /////////////////////////////////////// diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java new file mode 100644 index 0000000000000..6a938a9e4b17b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.io.PrintStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.EnumSet; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSUtilClient; + +/** + * Metadata about a snapshottable directory. + */ +public class SnapshotStatus { + /** + * Basic information of the snapshot directory. + */ + private final HdfsFileStatus dirStatus; + + /** + * Snapshot ID for the snapshot. + */ + private final int snapshotID; + + /** + * Full path of the parent. + */ + private byte[] parentFullPath; + + public SnapshotStatus(long modificationTime, long accessTime, + FsPermission permission, + EnumSet flags, + String owner, String group, byte[] localName, + long inodeId, int childrenNum, int snapshotID, + byte[] parentFullPath) { + this.dirStatus = new HdfsFileStatus.Builder() + .isdir(true) + .mtime(modificationTime) + .atime(accessTime) + .perm(permission) + .flags(flags) + .owner(owner) + .group(group) + .path(localName) + .fileId(inodeId) + .children(childrenNum) + .build(); + this.snapshotID = snapshotID; + this.parentFullPath = parentFullPath; + } + + public SnapshotStatus(HdfsFileStatus dirStatus, + int snapshotNumber, byte[] parentFullPath) { + this.dirStatus = dirStatus; + this.snapshotID = snapshotNumber; + this.parentFullPath = parentFullPath; + } + + /** + * sets the prent path name. + * @param path parent path + */ + public void setParentFullPath(byte[] path) { + parentFullPath = path; + } + + /** + * @return snapshot id for the snapshot + */ + public int getSnapshotID() { + return snapshotID; + } + + /** + * @return The basic information of the directory + */ + public HdfsFileStatus getDirStatus() { + return dirStatus; + } + + /** + * @return Full path of the file + */ + public byte[] getParentFullPath() { + return parentFullPath; + } + + /** + * @return Full path of the snapshot + */ + public Path getFullPath() { + String parentFullPathStr = + (parentFullPath == null || parentFullPath.length == 0) ? + "/" : DFSUtilClient.bytes2String(parentFullPath); + return new Path(getSnapshotPath(parentFullPathStr, + dirStatus.getLocalName())); + } + + /** + * Print a list of {@link SnapshotStatus} out to a given stream. + * + * @param stats The list of {@link SnapshotStatus} + * @param out The given stream for printing. + */ + public static void print(SnapshotStatus[] stats, + PrintStream out) { + if (stats == null || stats.length == 0) { + out.println(); + return; + } + int maxRepl = 0, maxLen = 0, maxOwner = 0, maxGroup = 0; + int maxSnapshotID = 0; + for (SnapshotStatus status : stats) { + maxRepl = maxLength(maxRepl, status.dirStatus.getReplication()); + maxLen = maxLength(maxLen, status.dirStatus.getLen()); + maxOwner = maxLength(maxOwner, status.dirStatus.getOwner()); + maxGroup = maxLength(maxGroup, status.dirStatus.getGroup()); + maxSnapshotID = maxLength(maxSnapshotID, status.snapshotID); + } + + String lineFormat = "%s%s " // permission string + + "%" + maxRepl + "s " + + (maxOwner > 0 ? "%-" + maxOwner + "s " : "%s") + + (maxGroup > 0 ? "%-" + maxGroup + "s " : "%s") + + "%" + maxLen + "s " + + "%s " // mod time + + "%" + maxSnapshotID + "s " + + "%s"; // path + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); + + for (SnapshotStatus status : stats) { + String line = String.format(lineFormat, "d", + status.dirStatus.getPermission(), + status.dirStatus.getReplication(), + status.dirStatus.getOwner(), + status.dirStatus.getGroup(), + String.valueOf(status.dirStatus.getLen()), + dateFormat.format(new Date(status.dirStatus.getModificationTime())), + status.snapshotID, + getSnapshotPath(DFSUtilClient.bytes2String(status.parentFullPath), + status.dirStatus.getLocalName()) + ); + out.println(line); + } + } + + private static int maxLength(int n, Object value) { + return Math.max(n, String.valueOf(value).length()); + } + + public static class Bean { + private final String path; + private final int snapshotID; + private final long modificationTime; + private final short permission; + private final String owner; + private final String group; + + public Bean(String path, int snapshotID, long + modificationTime, short permission, String owner, String group) { + this.path = path; + this.snapshotID = snapshotID; + this.modificationTime = modificationTime; + this.permission = permission; + this.owner = owner; + this.group = group; + } + + public String getPath() { + return path; + } + + public int getSnapshotID() { + return snapshotID; + } + + public long getModificationTime() { + return modificationTime; + } + + public short getPermission() { + return permission; + } + + public String getOwner() { + return owner; + } + + public String getGroup() { + return group; + } + } + + static String getSnapshotPath(String snapshottableDir, + String snapshotRelativePath) { + String parentFullPathStr = + snapshottableDir == null || snapshottableDir.isEmpty() ? + "/" : snapshottableDir; + final StringBuilder b = new StringBuilder(parentFullPathStr); + if (b.charAt(b.length() - 1) != Path.SEPARATOR_CHAR) { + b.append(Path.SEPARATOR); + } + return b.append(HdfsConstants.DOT_SNAPSHOT_DIR) + .append(Path.SEPARATOR) + .append(snapshotRelativePath) + .toString(); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 7e41460ca4c63..0674cefe2e2df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -89,6 +89,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusRequestProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.ModifyAclEntriesRequestProto; @@ -150,6 +151,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePolicyRequestProto; @@ -1299,6 +1302,25 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() } } + @Override + public SnapshotStatus[] getSnapshotListing(String path) + throws IOException { + GetSnapshotListingRequestProto req = + GetSnapshotListingRequestProto.newBuilder() + .setSnapshotRoot(path).build(); + try { + GetSnapshotListingResponseProto result = rpcProxy + .getSnapshotListing(null, req); + + if (result.hasSnapshotList()) { + return PBHelperClient.convert(result.getSnapshotList()); + } + return null; + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String fromSnapshot, String toSnapshot) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 9fc302464271d..efe020a3acf9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -114,6 +114,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto; @@ -184,6 +185,8 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotListingProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotStatusProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto; @@ -1669,6 +1672,48 @@ public static SnapshottableDirectoryStatus convert( sdirStatusProto.getParentFullpath().toByteArray()); } + public static SnapshotStatus[] convert( + HdfsProtos.SnapshotListingProto sdlp) { + if (sdlp == null) { + return null; + } + List list = sdlp + .getSnapshotListingList(); + if (list.isEmpty()) { + return new SnapshotStatus[0]; + } else { + SnapshotStatus[] result = + new SnapshotStatus[list.size()]; + for (int i = 0; i < list.size(); i++) { + result[i] = convert(list.get(i)); + } + return result; + } + } + + public static SnapshotStatus convert( + HdfsProtos.SnapshotStatusProto sdirStatusProto) { + if (sdirStatusProto == null) { + return null; + } + final HdfsFileStatusProto status = sdirStatusProto.getDirStatus(); + EnumSet flags = status.hasFlags() + ? convertFlags(status.getFlags()) + : convertFlags(status.getPermission()); + return new SnapshotStatus( + status.getModificationTime(), + status.getAccessTime(), + convert(status.getPermission()), + flags, + status.getOwner(), + status.getGroup(), + status.getPath().toByteArray(), + status.getFileId(), + status.getChildrenNum(), + sdirStatusProto.getSnapshotID(), + sdirStatusProto.getParentFullpath().toByteArray()); + } + // DataEncryptionKey public static DataEncryptionKey convert(DataEncryptionKeyProto bet) { String encryptionAlgorithm = bet.getEncryptionAlgorithm(); @@ -2367,6 +2412,23 @@ public static SnapshottableDirectoryStatusProto convert( return builder.build(); } + public static HdfsProtos.SnapshotStatusProto convert(SnapshotStatus status) { + if (status == null) { + return null; + } + byte[] parentFullPath = status.getParentFullPath(); + ByteString parentFullPathBytes = getByteString( + parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath); + HdfsFileStatusProto fs = convert(status.getDirStatus()); + HdfsProtos.SnapshotStatusProto.Builder builder = + HdfsProtos.SnapshotStatusProto + .newBuilder() + .setSnapshotID(status.getSnapshotID()) + .setParentFullpath(parentFullPathBytes) + .setDirStatus(fs); + return builder.build(); + } + public static HdfsFileStatusProto[] convert(HdfsFileStatus[] fs) { if (fs == null) return null; final int len = fs.length; @@ -2649,6 +2711,21 @@ public static SnapshottableDirectoryListingProto convert( .addAllSnapshottableDirListing(protoList).build(); } + public static HdfsProtos.SnapshotListingProto convert( + SnapshotStatus[] status) { + if (status == null) { + return null; + } + HdfsProtos.SnapshotStatusProto[] protos = + new HdfsProtos.SnapshotStatusProto[status.length]; + for (int i = 0; i < status.length; i++) { + protos[i] = convert(status[i]); + } + List protoList = Arrays.asList(protos); + return SnapshotListingProto.newBuilder() + .addAllSnapshotListing(protoList).build(); + } + public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { if (entry == null) { return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index 3fb57bc02d0ac..20967cc13ab86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -303,6 +303,14 @@ message GetSnapshottableDirListingResponseProto { optional SnapshottableDirectoryListingProto snapshottableDirList = 1; } +message GetSnapshotListingRequestProto { + required string snapshotRoot = 1; +} + +message GetSnapshotListingResponseProto { + optional SnapshotListingProto snapshotList = 1; +} + message GetSnapshotDiffReportRequestProto { required string snapshotRoot = 1; required string fromSnapshot = 2; @@ -986,6 +994,8 @@ service ClientNamenodeProtocol { returns(DisallowSnapshotResponseProto); rpc getSnapshottableDirListing(GetSnapshottableDirListingRequestProto) returns(GetSnapshottableDirListingResponseProto); + rpc getSnapshotListing(GetSnapshotListingRequestProto) + returns(GetSnapshotListingResponseProto); rpc deleteSnapshot(DeleteSnapshotRequestProto) returns(DeleteSnapshotResponseProto); rpc getSnapshotDiffReport(GetSnapshotDiffReportRequestProto) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 82fe329c9ce5e..3e24d73ce2d26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -563,6 +563,20 @@ message SnapshottableDirectoryStatusProto { required bytes parent_fullpath = 4; } +/** + * Status of a snapshot directory: besides the normal information for + * a directory status, also include snapshot ID, and + * the full path of the parent directory. + */ +message SnapshotStatusProto { + required HdfsFileStatusProto dirStatus = 1; + + // Fields specific for snapshot directory + required uint32 snapshotID = 2; + required bytes parent_fullpath = 3; +} + + /** * Snapshottable directory listing */ @@ -570,6 +584,13 @@ message SnapshottableDirectoryListingProto { repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1; } +/** + * Snapshot listing + */ +message SnapshotListingProto { + repeated SnapshotStatusProto snapshotListing = 1; +} + /** * Snapshot diff report entry */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java index 41069b439784f..4e6f4e3f4ba38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java @@ -41,6 +41,7 @@ public class TestReadOnly { "getListing", "getBatchedListing", "getSnapshottableDirListing", + "getSnapshotListing", "getPreferredBlockSize", "listCorruptFileBlocks", "getFileInfo", diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index 680cdc93250ec..e2ec0303333f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; @@ -1314,6 +1315,12 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return snapshotProto.getSnapshottableDirListing(); } + @Override + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + return snapshotProto.getSnapshotListing(snapshotRoot); + } + @Override public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 5905a1dbbd370..97b146c947442 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -112,6 +112,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.NamenodeProtocolService; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol; @@ -1130,6 +1131,12 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return clientProto.getSnapshottableDirListing(); } + @Override // ClientProtocol + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + return clientProto.getSnapshotListing(snapshotRoot); + } + @Override // ClientProtocol public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java index 7b08092d6431a..3f4f4cb46bd82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java @@ -24,10 +24,12 @@ import java.util.Map.Entry; import java.util.Set; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; @@ -157,6 +159,39 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return RouterRpcServer.merge(ret, SnapshottableDirectoryStatus.class); } + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + rpcServer.checkOperation(NameNode.OperationCategory.READ); + final List locations = + rpcServer.getLocationsForPath(snapshotRoot, true, false); + RemoteMethod remoteMethod = new RemoteMethod("getSnapshotListing", + new Class[]{String.class}, + new RemoteParam()); + SnapshotStatus[] response; + if (rpcServer.isInvokeConcurrent(snapshotRoot)) { + Map ret = rpcClient.invokeConcurrent( + locations, remoteMethod, true, false, SnapshotStatus[].class); + response = ret.values().iterator().next(); + String src = ret.keySet().iterator().next().getSrc(); + String dst = ret.keySet().iterator().next().getDest(); + for (SnapshotStatus s : response) { + String mountPath = DFSUtil.bytes2String(s.getParentFullPath()). + replaceFirst(src, dst); + s.setParentFullPath(DFSUtil.string2Bytes(mountPath)); + } + } else { + response = rpcClient.invokeSequential( + locations, remoteMethod, SnapshotStatus[].class, null); + RemoteLocation loc = locations.get(0); + for (SnapshotStatus s : response) { + String mountPath = DFSUtil.bytes2String(s.getParentFullPath()). + replaceFirst(loc.getDest(), loc.getSrc()); + s.setParentFullPath(DFSUtil.string2Bytes(mountPath)); + } + } + return response; + } + public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index b9b1212333eea..db118f56234d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -84,8 +84,6 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -94,6 +92,9 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; @@ -110,6 +111,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; @@ -926,6 +928,16 @@ public void testGetSnapshotListing() throws IOException { SnapshottableDirectoryStatus snapshotDir0 = dirList[0]; assertEquals(snapshotPath, snapshotDir0.getFullPath().toString()); + // check for snapshot listing through the Router + SnapshotStatus[] snapshots = routerProtocol. + getSnapshotListing(snapshotPath); + assertEquals(2, snapshots.length); + assertEquals(SnapshotTestHelper.getSnapshotRoot( + new Path(snapshotPath), snapshot1), + snapshots[0].getFullPath()); + assertEquals(SnapshotTestHelper.getSnapshotRoot( + new Path(snapshotPath), snapshot2), + snapshots[1].getFullPath()); // Check for difference report in two snapshot SnapshotDiffReport diffReport = routerProtocol.getSnapshotDiffReport( snapshotPath, snapshot1, snapshot2); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 7a8bf8dbe0deb..fa933540735ca 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -54,6 +54,7 @@ function hadoop_usage hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode." hadoop_add_subcommand "journalnode" daemon "run the DFS journalnode" hadoop_add_subcommand "lsSnapshottableDir" client "list all snapshottable dirs owned by the current user" + hadoop_add_subcommand "lsSnapshot" client "list all snapshots for a snapshottable directory" hadoop_add_subcommand "mover" daemon "run a utility to move block replicas across storage types" hadoop_add_subcommand "namenode" daemon "run the DFS namenode" hadoop_add_subcommand "nfs3" daemon "run an NFS version 3 gateway" @@ -166,6 +167,9 @@ function hdfscmd_case lsSnapshottableDir) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir ;; + lsSnapshot) + HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshot + ;; mover) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd index 23d6a5aa1c301..21d4de75cc8b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd @@ -59,7 +59,7 @@ if "%1" == "--loglevel" ( ) ) - set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck fsImageValidation balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto dfsrouter dfsrouteradmin debug + set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck fsImageValidation balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir lsSnapshot cacheadmin mover storagepolicies classpath crypto dfsrouter dfsrouteradmin debug for %%i in ( %hdfscommands% ) do ( if %hdfs-command% == %%i set hdfscommand=true ) @@ -167,6 +167,10 @@ goto :eof set CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir goto :eof +:lsSnapshot + set CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshot + goto :eof + :cacheadmin set CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin goto :eof @@ -253,6 +257,8 @@ goto :eof @echo current directory contents with a snapshot @echo lsSnapshottableDir list all snapshottable dirs owned by the current user @echo Use -help to see options + @echo lsSnapshot list all snapshots for a snapshottable dir + @echo Use -help to see options @echo cacheadmin configure the HDFS cache @echo crypto configure HDFS encryption zones @echo mover run a utility to move block replicas across storage types diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index e0afe006a2f9a..5132afaa4b15c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusRequestProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto; @@ -161,6 +162,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePolicyRequestProto; @@ -325,6 +328,9 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements static final GetSnapshottableDirListingResponseProto NULL_GET_SNAPSHOTTABLE_DIR_LISTING_RESPONSE = GetSnapshottableDirListingResponseProto.newBuilder().build(); + static final GetSnapshotListingResponseProto + NULL_GET_SNAPSHOT_LISTING_RESPONSE = + GetSnapshotListingResponseProto.newBuilder().build(); static final SetStoragePolicyResponseProto VOID_SET_STORAGE_POLICY_RESPONSE = SetStoragePolicyResponseProto.newBuilder().build(); static final UnsetStoragePolicyResponseProto @@ -1349,6 +1355,24 @@ public GetSnapshottableDirListingResponseProto getSnapshottableDirListing( } } + @Override + public GetSnapshotListingResponseProto getSnapshotListing( + RpcController controller, GetSnapshotListingRequestProto request) + throws ServiceException { + try { + SnapshotStatus[] result = server + .getSnapshotListing(request.getSnapshotRoot()); + if (result != null) { + return GetSnapshotListingResponseProto.newBuilder(). + setSnapshotList(PBHelperClient.convert(result)).build(); + } else { + return NULL_GET_SNAPSHOT_LISTING_RESPONSE; + } + } catch (IOException e) { + throw new ServiceException(e); + } + } + @Override public GetSnapshotDiffReportResponseProto getSnapshotDiffReport( RpcController controller, GetSnapshotDiffReportRequestProto request) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java index 923c6a88b0318..f264dc34063f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; @@ -154,6 +155,22 @@ static SnapshottableDirectoryStatus[] getSnapshottableDirListing( } } + static SnapshotStatus[] getSnapshotListing( + FSDirectory fsd, FSPermissionChecker pc, SnapshotManager snapshotManager, + String path) + throws IOException { + fsd.readLock(); + try { + INodesInPath iip = fsd.getINodesInPath(path, DirOp.READ); + if (fsd.isPermissionEnabled()) { + fsd.checkPathAccess(pc, iip, FsAction.READ); + } + return snapshotManager.getSnapshotListing(iip); + } finally { + fsd.readUnlock(); + } + } + static SnapshotDiffReport getSnapshotDiffReport(FSDirectory fsd, FSPermissionChecker pc, SnapshotManager snapshotManager, String path, String fromSnapshot, String toSnapshot) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index fe39b071e207c..9efcab2872748 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -99,6 +99,7 @@ import org.apache.commons.text.CaseUtils; import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; import static org.apache.hadoop.ha.HAServiceProtocol.HAServiceState.ACTIVE; @@ -7001,7 +7002,38 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() logAuditEvent(true, operationName, null, null, null); return status; } - + + /** + * Get the list of snapshots for a given snapshottable directory. + * + * @return The list of all the snapshots for a snapshottable directory + * @throws IOException + */ + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + final String operationName = "listSnapshotDirectory"; + SnapshotStatus[] status; + checkOperation(OperationCategory.READ); + boolean success = false; + final FSPermissionChecker pc = getPermissionChecker(); + FSPermissionChecker.setOperationType(operationName); + try { + readLock(); + try { + checkOperation(OperationCategory.READ); + status = FSDirSnapshotOp.getSnapshotListing(dir, pc, snapshotManager, + snapshotRoot); + success = true; + } finally { + readUnlock(operationName, getLockReportInfoSupplier(null)); + } + } catch (AccessControlException ace) { + logAuditEvent(success, "listSnapshots", snapshotRoot); + throw ace; + } + logAuditEvent(success, "listSnapshots", snapshotRoot); + return status; + } /** * Get the difference between two snapshots (or between a snapshot and the * current status) of a snapshottable directory. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 230e4020117f0..b42252d9aa9d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -132,6 +132,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol; @@ -2004,6 +2005,16 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() return status; } + @Override // Client Protocol + public SnapshotStatus[] getSnapshotListing(String snapshotRoot) + throws IOException { + checkNNStartup(); + SnapshotStatus[] status = namesystem + .getSnapshotListing(snapshotRoot); + metrics.incrListSnapshotsOps(); + return status; + } + @Override // ClientProtocol public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, String earlierSnapshotName, String laterSnapshotName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index c15cdbdd48e4e..de99ddfaa92d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -77,6 +77,8 @@ public class NameNodeMetrics { MutableCounterLong renameSnapshotOps; @Metric("Number of listSnapshottableDirectory operations") MutableCounterLong listSnapshottableDirOps; + @Metric("Number of listSnapshots operations") + MutableCounterLong listSnapshotOps; @Metric("Number of snapshotDiffReport operations") MutableCounterLong snapshotDiffReportOps; @Metric("Number of blockReceivedAndDeleted calls") @@ -106,6 +108,7 @@ public long totalFileOps(){ disallowSnapshotOps.value() + renameSnapshotOps.value() + listSnapshottableDirOps.value() + + listSnapshotOps.value() + createSymlinkOps.value() + snapshotDiffReportOps.value(); } @@ -319,6 +322,10 @@ public void incrRenameSnapshotOps() { public void incrListSnapshottableDirOps() { listSnapshottableDirOps.incr(); } + + public void incrListSnapshotsOps() { + listSnapshotOps.incr(); + } public void incrSnapshotDiffReportOps() { snapshotDiffReportOps.incr(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 9ace8a97b8641..e9729abd42347 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -50,10 +50,18 @@ import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotInfo; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.namenode.*; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodesInPath; +import org.apache.hadoop.hdfs.server.namenode.LeaseManager; +import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.metrics2.util.MBeans; import com.google.common.base.Preconditions; @@ -501,7 +509,36 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing( return statusList.toArray( new SnapshottableDirectoryStatus[statusList.size()]); } - + + /** + * List all the snapshots under a snapshottable directory. + */ + public SnapshotStatus[] getSnapshotListing(INodesInPath iip) + throws IOException { + INodeDirectory srcRoot = getSnapshottableRoot(iip); + ReadOnlyList snapshotList = srcRoot. + getDirectorySnapshottableFeature().getSnapshotList(); + SnapshotStatus[] statuses = new SnapshotStatus[snapshotList.size()]; + for (int count = 0; count < snapshotList.size(); count++) { + Snapshot s = snapshotList.get(count); + Snapshot.Root dir = s.getRoot(); + statuses[count] = new SnapshotStatus(dir.getModificationTime(), + dir.getAccessTime(), dir.getFsPermission(), + EnumSet.noneOf(HdfsFileStatus.Flags.class), + dir.getUserName(), dir.getGroupName(), + dir.getLocalNameBytes(), dir.getId(), + // the children number is same as the + // live fs as the children count is not cached per snashot. + // It is just used here to construct the HdfsFileStatus object. + // It is expensive to build the snapshot tree for the directory + // and determine the child count. + dir.getChildrenNum(Snapshot.CURRENT_STATE_ID), + s.getId(), DFSUtil.string2Bytes(dir.getParent().getFullPathName())); + + } + return statuses; + } + /** * Compute the difference between two snapshots of a directory, or between a * snapshot of the directory and its current tree. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java index 27cdf70279087..59a6c8deed9e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java @@ -43,7 +43,7 @@ public class AdminHelper { static final int MAX_LINE_WIDTH = 80; static final String HELP_COMMAND_NAME = "-help"; - static DistributedFileSystem getDFS(Configuration conf) + public static DistributedFileSystem getDFS(Configuration conf) throws IOException { FileSystem fs = FileSystem.get(conf); return checkAndGetDFS(fs, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshot.java new file mode 100644 index 0000000000000..65b38afe416eb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshot.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.snapshot; + + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; +import org.apache.hadoop.hdfs.tools.AdminHelper; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * A tool used to list all snapshottable directories that are owned by the + * current user. The tool returns all the snapshottable directories if the user + * is a super user. + */ +@InterfaceAudience.Private +public class LsSnapshot extends Configured implements Tool { + @Override + public int run(String[] argv) throws Exception { + String description = "hdfs lsSnapshot : \n" + + "\tGet the list of snapshots for a snapshottable directory.\n"; + + if(argv.length != 1) { + System.err.println("Invalid no of arguments"); + System.err.println("Usage: \n" + description); + return 1; + } + Path snapshotRoot = new Path(argv[0]); + try { + DistributedFileSystem dfs = AdminHelper.getDFS(getConf()); + SnapshotStatus[] stats = dfs.getSnapshotListing(snapshotRoot); + SnapshotStatus.print(stats, System.out); + } catch (Exception e) { + String[] content = e.getLocalizedMessage().split("\n"); + System.err.println("lsSnapshot: " + content[0]); + return 1; + } + return 0; + } + public static void main(String[] argv) throws Exception { + int rc = ToolRunner.run(new LsSnapshot(), argv); + System.exit(rc); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index d199c06afb740..4b7a7a751049c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -158,6 +158,16 @@ Usage: `hdfs lsSnapshottableDir [-help]` Get the list of snapshottable directories. When this is run as a super user, it returns all snapshottable directories. Otherwise it returns those directories that are owned by the current user. +### `lsSnapshot` + +Usage: `hdfs lsSnapshot [-help]` + +| COMMAND\_OPTION | Description | +|:---- |:---- | +| `-help` | print help | + +Get the list of snapshots for a snapshottable directory. + ### `jmxget` Usage: `hdfs jmxget [-localVM ConnectorURL | -port port | -server mbeanserver | -service service]` diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md index af55f33b2640c..1b9b573886b0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md @@ -236,6 +236,23 @@ See also the corresponding Java API `SnapshottableDirectoryStatus[] getSnapshottableDirectoryListing()` in `DistributedFileSystem`. +#### Get Snapshot Listing + +Get all the snapshots for a snapshottable directory. + +* Command: + + hdfs lsSnapshot + +* Arguments: + + | --- | --- | + | path | The path of the snapshottable directory. | + +See also the corresponding Java API +`SnapshotStatus[] getSnapshotListing()` +in `DistributedFileSystem`. + #### Get Snapshots Difference Report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java new file mode 100644 index 0000000000000..dcff2f7f7c69c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.SnapshotException; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; +import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.test.LambdaTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Tests listSnapshot. + */ +public class TestListSnapshot { + + static final short REPLICATION = 3; + + private final Path dir1 = new Path("/TestSnapshot1"); + + Configuration conf; + MiniDFSCluster cluster; + FSNamesystem fsn; + DistributedFileSystem hdfs; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) + .build(); + cluster.waitActive(); + fsn = cluster.getNamesystem(); + hdfs = cluster.getFileSystem(); + hdfs.mkdirs(dir1); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + /** + * Test listing all the snapshottable directories. + */ + @Test(timeout = 60000) + public void testListSnapshot() throws Exception { + fsn.getSnapshotManager().setAllowNestedSnapshots(true); + + // Initially there is no snapshottable directories in the system + SnapshotStatus[] snapshotStatuses = null; + SnapshottableDirectoryStatus[] dirs = hdfs.getSnapshottableDirListing(); + assertNull(dirs); + LambdaTestUtils.intercept(SnapshotException.class, + "Directory is not a " + "snapshottable directory", + () -> hdfs.getSnapshotListing(dir1)); + // Make root as snapshottable + final Path root = new Path("/"); + hdfs.allowSnapshot(root); + dirs = hdfs.getSnapshottableDirListing(); + assertEquals(1, dirs.length); + assertEquals("", dirs[0].getDirStatus().getLocalName()); + assertEquals(root, dirs[0].getFullPath()); + snapshotStatuses = hdfs.getSnapshotListing(root); + assertTrue(snapshotStatuses.length == 0); + // Make root non-snaphsottable + hdfs.disallowSnapshot(root); + dirs = hdfs.getSnapshottableDirListing(); + assertNull(dirs); + snapshotStatuses = hdfs.getSnapshotListing(root); + assertTrue(snapshotStatuses.length == 0); + + // Make dir1 as snapshottable + hdfs.allowSnapshot(dir1); + hdfs.createSnapshot(dir1, "s0"); + snapshotStatuses = hdfs.getSnapshotListing(dir1); + assertEquals(1, snapshotStatuses.length); + assertEquals("s0", snapshotStatuses[0].getDirStatus(). + getLocalName()); + assertEquals(SnapshotTestHelper.getSnapshotRoot(dir1, "s0"), + snapshotStatuses[0].getFullPath()); + // snapshot id is zero + assertEquals(0, snapshotStatuses[0].getSnapshotID()); + // Create a snapshot for dir1 + hdfs.createSnapshot(dir1, "s1"); + hdfs.createSnapshot(dir1, "s2"); + snapshotStatuses = hdfs.getSnapshotListing(dir1); + // There are now 3 snapshots for dir1 + assertEquals(3, snapshotStatuses.length); + assertEquals("s0", snapshotStatuses[0].getDirStatus(). + getLocalName()); + assertEquals(SnapshotTestHelper.getSnapshotRoot(dir1, "s0"), + snapshotStatuses[0].getFullPath()); + assertEquals("s1", snapshotStatuses[1].getDirStatus(). + getLocalName()); + assertEquals(SnapshotTestHelper.getSnapshotRoot(dir1, "s1"), + snapshotStatuses[1].getFullPath()); + assertEquals("s2", snapshotStatuses[2].getDirStatus(). + getLocalName()); + assertEquals(SnapshotTestHelper.getSnapshotRoot(dir1, "s2"), + snapshotStatuses[2].getFullPath()); + hdfs.deleteSnapshot(dir1, "s2"); + snapshotStatuses = hdfs.getSnapshotListing(dir1); + // There are now 2 snapshots for dir1 + assertEquals(2, snapshotStatuses.length); + } +} \ No newline at end of file From 5d8600e80ad7864b332b60d5a01585fdf00848ee Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Wed, 29 Jul 2020 23:26:08 +0530 Subject: [PATCH 023/335] YARN-10369. Make NMTokenSecretManagerInRM sending NMToken for nodeId DEBUG. Contributed by Jim Brennan. --- .../resourcemanager/security/NMTokenSecretManagerInRM.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java index f702c71448c85..4c7b7ee7584d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java @@ -197,8 +197,8 @@ public NMToken createAndGetNMToken(String applicationSubmitter, NMToken nmToken = null; if (nodeSet != null) { if (!nodeSet.contains(container.getNodeId())) { - LOG.info("Sending NMToken for nodeId : " + container.getNodeId() - + " for container : " + container.getId()); + LOG.debug("Sending NMToken for nodeId : {} for container : {}", + container.getNodeId(), container.getId()); Token token = createNMToken(container.getId().getApplicationAttemptId(), container.getNodeId(), applicationSubmitter); From cf4eb75608527a8a56ad9d13a3009ccc38b29c8e Mon Sep 17 00:00:00 2001 From: ywheel Date: Thu, 30 Jul 2020 12:01:22 +0800 Subject: [PATCH 024/335] MAPREDUCE-7051. Fix typo in MultipleOutputFormat (#338) --- .../org/apache/hadoop/mapred/lib/MultipleOutputFormat.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputFormat.java index 90ce57aa36cc8..125c7568c289a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputFormat.java @@ -175,7 +175,7 @@ protected V generateActualValue(K key, V value) { /** - * Generate the outfile name based on a given anme and the input file name. If + * Generate the outfile name based on a given name and the input file name. If * the {@link JobContext#MAP_INPUT_FILE} does not exists (i.e. this is not for a map only job), * the given name is returned unchanged. If the config value for * "num.of.trailing.legs.to.use" is not set, or set 0 or negative, the given @@ -187,7 +187,7 @@ protected V generateActualValue(K key, V value) { * the job config * @param name * the output file name - * @return the outfile name based on a given anme and the input file name. + * @return the outfile name based on a given name and the input file name. */ protected String getInputFileBasedOutputFileName(JobConf job, String name) { String infilepath = job.get(MRJobConfig.MAP_INPUT_FILE); From e0c9653166df48a47267dbc81d124ab78267e039 Mon Sep 17 00:00:00 2001 From: Eric E Payne Date: Thu, 30 Jul 2020 15:30:22 +0000 Subject: [PATCH 025/335] YARN-1529: Add Localization overhead metrics to NM. Contributed by Jim_Brennan. --- .../hadoop/yarn/api/ApplicationConstants.java | 9 ++++ .../ContainerManagerImpl.java | 2 +- .../containermanager/container/Container.java | 10 ++++ .../container/ContainerImpl.java | 51 +++++++++++++++++++ .../ContainerResourceLocalizedEvent.java | 13 +++++ .../launcher/ContainerLaunch.java | 3 ++ .../localizer/LocalizedResource.java | 12 +++-- .../metrics/NodeManagerMetrics.java | 50 ++++++++++++++++++ .../TestContainerManager.java | 40 +++++++++++++++ .../launcher/TestContainerLaunch.java | 1 + .../nodemanager/webapp/MockContainer.java | 5 ++ 11 files changed, 191 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index f5d8f02cb9a6b..533eaddabb4f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -224,6 +224,15 @@ enum Environment { @Private CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"), + /** + * $LOCALIZATION_COUNTERS + * + * Since NM does not RPC Container JVM's we pass Localization counter + * vector as an environment variable + * + */ + LOCALIZATION_COUNTERS("LOCALIZATION_COUNTERS"), + /** * $CONTAINER_ID * Final, exported by NodeManager and non-modifiable by users. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 1758028b60408..3699d974bd49f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -212,7 +212,7 @@ private enum ReInitOp { private final ResourceLocalizationService rsrcLocalizationSrvc; private final AbstractContainersLauncher containersLauncher; private final AuxServices auxiliaryServices; - private final NodeManagerMetrics metrics; + @VisibleForTesting final NodeManagerMetrics metrics; protected final NodeStatusUpdater nodeStatusUpdater; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index f8bb9b00b9493..0c06aae95327b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -138,4 +138,14 @@ T getContainerRuntimeData(Class runtimeClazz) * @return localization statuses. */ List getLocalizationStatuses(); + + /** + * Vector of localization counters to be passed from NM to application + * container via environment variable {@code $LOCALIZATION_COUNTERS}. See + * {@link org.apache.hadoop.yarn.api.ApplicationConstants.Environment#LOCALIZATION_COUNTERS} + * + * @return coma-separated counter values + */ + String localizationCountersAsString(); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index f253b041d328b..5a4d20c373728 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -100,6 +101,14 @@ import org.apache.hadoop.yarn.util.resource.Resources; public class ContainerImpl implements Container { + private enum LocalizationCounter { + // 1-to-1 correspondence with MR TaskCounter.LOCALIZED_* + BYTES_MISSED, + BYTES_CACHED, + FILES_MISSED, + FILES_CACHED, + MILLIS; + } private static final class ReInitializationContext { private final ContainerLaunchContext newLaunchContext; @@ -153,6 +162,9 @@ private ReInitializationContext createContextForRollback() { private final NMStateStoreService stateStore; private final Credentials credentials; private final NodeManagerMetrics metrics; + private final long[] localizationCounts = + new long[LocalizationCounter.values().length]; + private volatile ContainerLaunchContext launchContext; private volatile ContainerTokenIdentifier containerTokenIdentifier; private final ContainerId containerId; @@ -1211,6 +1223,12 @@ public ContainerState transition(ContainerImpl container, } container.containerLocalizationStartTime = clock.getTime(); + // duration = end - start; + // record in RequestResourcesTransition: -start + // add in LocalizedTransition: +end + // + container.localizationCounts[LocalizationCounter.MILLIS.ordinal()] + = -Time.monotonicNow(); // Send requests for public, private resources Map cntrRsrc; @@ -1259,6 +1277,21 @@ public ContainerState transition(ContainerImpl container, return ContainerState.LOCALIZING; } + final long localizedSize = rsrcEvent.getSize(); + if (localizedSize > 0) { + container.localizationCounts + [LocalizationCounter.BYTES_MISSED.ordinal()] += localizedSize; + container.localizationCounts + [LocalizationCounter.FILES_MISSED.ordinal()]++; + } else if (localizedSize < 0) { + // cached: recorded negative, restore the sign + container.localizationCounts + [LocalizationCounter.BYTES_CACHED.ordinal()] -= localizedSize; + container.localizationCounts + [LocalizationCounter.FILES_CACHED.ordinal()]++; + } + container.metrics.localizationCacheHitMiss(localizedSize); + // check to see if this resource should be uploaded to the shared cache // as well if (shouldBeUploadedToSharedCache(container, resourceRequest)) { @@ -1269,6 +1302,14 @@ public ContainerState transition(ContainerImpl container, return ContainerState.LOCALIZING; } + // duration = end - start; + // record in RequestResourcesTransition: -start + // add in LocalizedTransition: +end + // + container.localizationCounts[LocalizationCounter.MILLIS.ordinal()] + += Time.monotonicNow(); + container.metrics.localizationComplete( + container.localizationCounts[LocalizationCounter.MILLIS.ordinal()]); container.dispatcher.getEventHandler().handle( new ContainerLocalizationEvent(LocalizationEventType. CONTAINER_RESOURCES_LOCALIZED, container)); @@ -2301,4 +2342,14 @@ public T getContainerRuntimeData(Class runtimeClass) } return runtimeClass.cast(containerRuntimeData); } + + @Override + public String localizationCountersAsString() { + StringBuilder result = + new StringBuilder(String.valueOf(localizationCounts[0])); + for (int i = 1; i < localizationCounts.length; i++) { + result.append(',').append(localizationCounts[i]); + } + return result.toString(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceLocalizedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceLocalizedEvent.java index 4b742b14330e4..d5bcaa286e864 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceLocalizedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceLocalizedEvent.java @@ -25,6 +25,11 @@ public class ContainerResourceLocalizedEvent extends ContainerResourceEvent { private final Path loc; + // > 0: downloaded + // < 0: cached + // + private long size; + public ContainerResourceLocalizedEvent(ContainerId container, LocalResourceRequest rsrc, Path loc) { super(container, ContainerEventType.RESOURCE_LOCALIZED, rsrc); @@ -35,4 +40,12 @@ public Path getLocation() { return loc; } + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 41e26d4471362..33cfb22cd9a3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -1620,6 +1620,9 @@ public void sanitizeEnv(Map environment, Path pwd, addToEnvMap(environment, nmVars, Environment.PWD.name(), pwd.toString()); + addToEnvMap(environment, nmVars, Environment.LOCALIZATION_COUNTERS.name(), + container.localizationCountersAsString()); + if (!Shell.WINDOWS) { addToEnvMap(environment, nmVars, "JVM_PID", "$$"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java index a75a13e956b06..8a8a49ab04399 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java @@ -244,9 +244,11 @@ public void transition(LocalizedResource rsrc, ResourceEvent event) { Path.getPathWithoutSchemeAndAuthority(locEvent.getLocation()); rsrc.size = locEvent.getSize(); for (ContainerId container : rsrc.ref) { - rsrc.dispatcher.getEventHandler().handle( + final ContainerResourceLocalizedEvent localizedEvent = new ContainerResourceLocalizedEvent( - container, rsrc.rsrc, rsrc.localPath)); + container, rsrc.rsrc, rsrc.localPath); + localizedEvent.setSize(rsrc.size); + rsrc.dispatcher.getEventHandler().handle(localizedEvent); } } } @@ -281,9 +283,11 @@ public void transition(LocalizedResource rsrc, ResourceEvent event) { ResourceRequestEvent reqEvent = (ResourceRequestEvent) event; ContainerId container = reqEvent.getContext().getContainerId(); rsrc.ref.add(container); - rsrc.dispatcher.getEventHandler().handle( + final ContainerResourceLocalizedEvent localizedEvent = new ContainerResourceLocalizedEvent( - container, rsrc.rsrc, rsrc.localPath)); + container, rsrc.rsrc, rsrc.localPath); + localizedEvent.setSize(-rsrc.size); + rsrc.dispatcher.getEventHandler().handle(localizedEvent); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index 8ecc1a17ca727..d29dd6f6482a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -22,6 +22,7 @@ import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MutableCounterInt; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableGaugeFloat; @@ -98,6 +99,21 @@ public class NodeManagerMetrics { @Metric("Current CPU utilization") MutableGaugeFloat nodeCpuUtilization; + @Metric("Missed localization requests in bytes") + MutableCounterLong localizedCacheMissBytes; + @Metric("Cached localization requests in bytes") + MutableCounterLong localizedCacheHitBytes; + @Metric("Localization cache hit ratio (bytes)") + MutableGaugeInt localizedCacheHitBytesRatio; + @Metric("Missed localization requests (files)") + MutableCounterLong localizedCacheMissFiles; + @Metric("Cached localization requests (files)") + MutableCounterLong localizedCacheHitFiles; + @Metric("Localization cache hit ratio (files)") + MutableGaugeInt localizedCacheHitFilesRatio; + @Metric("Container localization time in milliseconds") + MutableRate localizationDurationMillis; + // CHECKSTYLE:ON:VisibilityModifier private JvmMetrics jvmMetrics = null; @@ -411,4 +427,38 @@ public float getNodeCpuUtilization() { public void setNodeCpuUtilization(float cpuUtilization) { this.nodeCpuUtilization.set(cpuUtilization); } + + private void updateLocalizationHitRatios() { + updateLocalizationHitRatio(localizedCacheHitBytes, localizedCacheMissBytes, + localizedCacheHitBytesRatio); + updateLocalizationHitRatio(localizedCacheHitFiles, localizedCacheMissFiles, + localizedCacheHitFilesRatio); + } + + private static void updateLocalizationHitRatio(MutableCounterLong hitCounter, + MutableCounterLong missedCounter, MutableGaugeInt ratioGauge) { + final long hits = hitCounter.value(); + final long misses = missedCounter.value(); + final long total = hits + misses; + if (total > 0) { + ratioGauge.set((int)(100 * hits / total)); + } + } + + public void localizationCacheHitMiss(long size) { + if (size > 0) { + localizedCacheMissBytes.incr(size); + localizedCacheMissFiles.incr(); + updateLocalizationHitRatios(); + } else if (size < 0) { + // cached: recorded negative, restore the sign + localizedCacheHitBytes.incr(-size); + localizedCacheHitFiles.incr(); + updateLocalizationHitRatios(); + } + } + + public void localizationComplete(long downloadMillis) { + localizationDurationMillis.add(downloadMillis); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 215223987f027..05014e45affca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -25,6 +25,10 @@ import org.apache.hadoop.yarn.api.records.LocalizationStatus; import org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.assertGauge; +import static org.apache.hadoop.test.MetricsAsserts.assertGaugeGt; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -43,6 +47,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -50,8 +55,10 @@ import java.util.function.Supplier; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service; import org.apache.hadoop.test.GenericTestUtils; @@ -321,6 +328,39 @@ public void testContainerSetup() throws Exception { BufferedReader reader = new BufferedReader(new FileReader(targetFile)); Assert.assertEquals("Hello World!", reader.readLine()); Assert.assertEquals(null, reader.readLine()); + + // + // check the localization counter + // + long targetFileSize = + FileUtil.getDU(targetFile.getCanonicalFile().getParentFile()); + MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics"); + assertCounter("LocalizedCacheMissBytes", targetFileSize, rb); + assertCounter("LocalizedCacheHitBytes", 0L, rb); + assertCounter("LocalizedCacheMissFiles", 1L, rb); + assertCounter("LocalizedCacheHitFiles", 0L, rb); + assertGaugeGt("LocalizationDurationMillisAvgTime", 0, rb); + assertGauge("LocalizedCacheHitBytesRatio", 0, rb); + assertGauge("LocalizedCacheHitFilesRatio", 0, rb); + + // test cache being used + final ContainerId cid1 = createContainerId(1); + containerManager.startContainers(StartContainersRequest.newInstance( + Collections.singletonList( + StartContainerRequest.newInstance( + containerLaunchContext, + createContainerToken(cid1, DUMMY_RM_IDENTIFIER, + context.getNodeId(), + user, + context.getContainerTokenSecretManager()))))); + waitForContainerState(containerManager, cid1, ContainerState.COMPLETE); + rb = getMetrics("NodeManagerMetrics"); + assertCounter("LocalizedCacheMissBytes", targetFileSize, rb); + assertCounter("LocalizedCacheHitBytes", targetFileSize, rb); + assertCounter("LocalizedCacheMissFiles", 1L, rb); + assertCounter("LocalizedCacheHitFiles", 1L, rb); + assertGauge("LocalizedCacheHitBytesRatio", 50, rb); + assertGauge("LocalizedCacheHitFilesRatio", 50, rb); } @Test (timeout = 10000L) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 283338315a05c..d42270561ed54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -865,6 +865,7 @@ private void verifyTailErrorLogOnContainerExit(Configuration conf, .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 1); when(container.getContainerId()).thenReturn(containerId); when(container.getUser()).thenReturn("test"); + when(container.localizationCountersAsString()).thenReturn(""); String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir( appId.toString(), containerId.toString()); Path containerLogDir = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index 4350bc0789a80..6e07fa5034d76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -123,6 +123,11 @@ public String toString() { return ""; } + @Override + public String localizationCountersAsString() { + return ""; + } + @Override public ResourceSet getResourceSet() { return null; From 05b3337a4605dcb6904cb3fe2a58e4dc424ef015 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 30 Jul 2020 10:36:51 -0700 Subject: [PATCH 026/335] HDFS-15481. Ordered snapshot deletion: garbage collect deleted snapshots (#2165) --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 - .../server/common/HdfsServerConstants.java | 2 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 6 +- .../hdfs/server/namenode/FSDirectory.java | 11 -- .../hdfs/server/namenode/FSNamesystem.java | 36 ++++ .../hdfs/server/namenode/INodeDirectory.java | 7 +- .../snapshot/AbstractINodeDiffList.java | 11 ++ .../DirectorySnapshottableFeature.java | 4 +- .../snapshot/FSImageFormatPBSnapshot.java | 2 +- .../server/namenode/snapshot/Snapshot.java | 7 + .../namenode/snapshot/SnapshotDeletionGc.java | 110 +++++++++++ .../namenode/snapshot/SnapshotManager.java | 117 ++++++++++- .../TestOrderedSnapshotDeletion.java | 64 ++++-- .../TestOrderedSnapshotDeletionGc.java | 184 ++++++++++++++++++ .../snapshot/TestSnapshotDeletion.java | 4 +- 15 files changed, 517 insertions(+), 53 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java rename hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/{ => snapshot}/TestOrderedSnapshotDeletion.java (76%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index dd24785688270..3865f9b4d72c5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -502,11 +502,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.snapshot.max.limit"; public static final int DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT = 65536; - public static final String DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED = - "dfs.namenode.snapshot.deletion.ordered"; - public static final boolean DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT - = false; - public static final String DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL = "dfs.namenode.snapshot.skiplist.interval"; public static final int DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL_DEFAULT = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index a55985edffcc5..3cd6f283f2a3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -366,7 +366,7 @@ enum BlockUCState { "security.hdfs.unreadable.by.superuser"; String XATTR_ERASURECODING_POLICY = "system.hdfs.erasurecoding.policy"; - String SNAPSHOT_XATTR_NAME = "system.hdfs.snapshot.deleted"; + String XATTR_SNAPSHOT_DELETED = "system.hdfs.snapshot.deleted"; String XATTR_SATISFY_STORAGE_POLICY = "user.hdfs.sps"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index 4f215acda1ba4..7e1657f48fce2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -44,7 +44,7 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO; -import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SNAPSHOT_XATTR_NAME; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SNAPSHOT_DELETED; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE; public class FSDirXAttrOp { @@ -328,10 +328,10 @@ public static INode unprotectedSetXAttrs( SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' on a file."); } - if (xaName.equals(SNAPSHOT_XATTR_NAME) && !(inode.isDirectory() && + if (xaName.equals(XATTR_SNAPSHOT_DELETED) && !(inode.isDirectory() && inode.getParent().isSnapshottable())) { throw new IOException("Can only set '" + - SNAPSHOT_XATTR_NAME + "' on a snapshot root."); + XATTR_SNAPSHOT_DELETED + "' on a snapshot root."); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 2c7932c0f8394..812b35f67d744 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -187,7 +187,6 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) { private boolean posixAclInheritanceEnabled; private final boolean xattrsEnabled; private final int xattrMaxSize; - private final boolean snapshotDeletionOrdered; // precision of access times. private final long accessTimePrecision; @@ -354,12 +353,6 @@ public enum DirOp { + " hard limit " + DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_HARD_LIMIT + ": (%s).", DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY); - this.snapshotDeletionOrdered = - conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, - DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT); - LOG.info("{} = {}", DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, - snapshotDeletionOrdered); - this.accessTimePrecision = conf.getLong( DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT); @@ -618,10 +611,6 @@ boolean isXattrsEnabled() { } int getXattrMaxSize() { return xattrMaxSize; } - public boolean isSnapshotDeletionOrdered() { - return snapshotDeletionOrdered; - } - boolean isAccessTimeSupported() { return accessTimePrecision > 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 9efcab2872748..72ece88b92a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -109,6 +109,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import com.google.common.collect.Maps; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotDeletionGc; import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.hdfs.protocol.BatchedDirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsPartialListing; @@ -493,6 +494,7 @@ private void logAuditEvent(boolean succeeded, FSDirectory dir; private BlockManager blockManager; private final SnapshotManager snapshotManager; + private final SnapshotDeletionGc snapshotDeletionGc; private final CacheManager cacheManager; private final DatanodeStatistics datanodeStatistics; @@ -974,6 +976,9 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { this.dtSecretManager = createDelegationTokenSecretManager(conf); this.dir = new FSDirectory(this, conf); this.snapshotManager = new SnapshotManager(conf, dir); + this.snapshotDeletionGc = snapshotManager.isSnapshotDeletionOrdered()? + new SnapshotDeletionGc(this, conf): null; + this.cacheManager = new CacheManager(this, conf, blockManager); // Init ErasureCodingPolicyManager instance. ErasureCodingPolicyManager.getInstance().init(conf); @@ -1360,6 +1365,10 @@ void startActiveServices() throws IOException { dir.enableQuotaChecks(); dir.ezManager.startReencryptThreads(); + if (snapshotDeletionGc != null) { + snapshotDeletionGc.schedule(); + } + if (haEnabled) { // Renew all of the leases before becoming active. // This is because, while we were in standby mode, @@ -5480,6 +5489,9 @@ private void registerMBean() { * Shutdown FSNamesystem. */ void shutdown() { + if (snapshotDeletionGc != null) { + snapshotDeletionGc.cancel(); + } if (snapshotManager != null) { snapshotManager.shutdown(); } @@ -7198,6 +7210,30 @@ void deleteSnapshot(String snapshotRoot, String snapshotName, logAuditEvent(true, operationName, rootPath, null, null); } + public void gcDeletedSnapshot(String snapshotRoot, String snapshotName) + throws IOException { + final String operationName = "gcDeletedSnapshot"; + String rootPath = null; + final INode.BlocksMapUpdateInfo blocksToBeDeleted; + + checkOperation(OperationCategory.WRITE); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); + rootPath = Snapshot.getSnapshotPath(snapshotRoot, snapshotName); + checkNameNodeSafeMode("Cannot gcDeletedSnapshot for " + rootPath); + + final long now = Time.now(); + final INodesInPath iip = dir.resolvePath(null, snapshotRoot, DirOp.WRITE); + snapshotManager.assertMarkedAsDeleted(iip, snapshotName); + blocksToBeDeleted = FSDirSnapshotOp.deleteSnapshot( + dir, snapshotManager, iip, snapshotName, now); + } finally { + writeUnlock(operationName, getLockReportInfoSupplier(rootPath)); + } + removeBlocks(blocksToBeDeleted); + } + /** * Remove a list of INodeDirectorySnapshottable from the SnapshotManager * @param toRemove the list of INodeDirectorySnapshottable to be removed diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index de710a1c876bc..5f44da8792b8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.util.ReadOnlyList; import com.google.common.annotations.VisibleForTesting; @@ -294,11 +295,11 @@ public Snapshot addSnapshot(int id, String name, * @param snapshotName Name of the snapshot. * @param mtime The snapshot deletion time set by Time.now(). */ - public Snapshot removeSnapshot( - ReclaimContext reclaimContext, String snapshotName, long mtime) + public Snapshot removeSnapshot(ReclaimContext reclaimContext, + String snapshotName, long mtime, SnapshotManager snapshotManager) throws SnapshotException { return getDirectorySnapshottableFeature().removeSnapshot( - reclaimContext, this, snapshotName, mtime); + reclaimContext, this, snapshotName, mtime, snapshotManager); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java index 163b181ed1892..776adf1defc54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java @@ -133,6 +133,17 @@ final void addFirst(D diff) { diff.setPosterior(first); } + /** @return the first diff. */ + final D getFirst() { + return diffs == null || diffs.isEmpty()? null: diffs.get(0); + } + + /** @return the first snapshot INode. */ + final A getFirstSnapshotINode() { + final D first = getFirst(); + return first == null? null: first.getSnapshotINode(); + } + /** @return the last diff. */ public final D getLast() { if (diffs == null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index fe97cf87015e7..18ef9a1c68134 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -239,7 +239,8 @@ public Snapshot addSnapshot(INodeDirectory snapshotRoot, int id, String name, */ public Snapshot removeSnapshot( INode.ReclaimContext reclaimContext, INodeDirectory snapshotRoot, - String snapshotName, long now) throws SnapshotException { + String snapshotName, long now, SnapshotManager snapshotManager) + throws SnapshotException { final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName)); if (i < 0) { throw new SnapshotException("Cannot delete snapshot " + snapshotName @@ -248,6 +249,7 @@ public Snapshot removeSnapshot( } else { final Snapshot snapshot = snapshotsByNames.get(i); int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId()); + snapshotManager.assertPrior(snapshotRoot, snapshotName, prior); snapshotRoot.cleanSubtree(reclaimContext, snapshot.getId(), prior); // remove from snapshotsByNames after successfully cleaning the subtree snapshotsByNames.remove(i); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java index 5f5cd41166a7e..097e3808c59aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java @@ -438,7 +438,7 @@ public void serializeSnapshotSection(OutputStream out) throws IOException { .setSnapshotCounter(sm.getSnapshotCounter()) .setNumSnapshots(sm.getNumSnapshots()); - INodeDirectory[] snapshottables = sm.getSnapshottableDirs(); + final List snapshottables = sm.getSnapshottableDirs(); for (INodeDirectory sdir : snapshottables) { b.addSnapshottableDir(sdir.getId()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java index f13ef611346a4..9a3ee2e6c4e61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java @@ -40,6 +40,8 @@ import org.apache.hadoop.security.AccessControlException; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SNAPSHOT_DELETED; + /** Snapshot of a sub-tree in the namesystem. */ @InterfaceAudience.Private public class Snapshot implements Comparable { @@ -156,6 +158,11 @@ static public class Root extends INodeDirectory { }).collect(Collectors.toList()).toArray(new Feature[0])); } + boolean isMarkedAsDeleted() { + final XAttrFeature f = getXAttrFeature(); + return f != null && f.getXAttr(XATTR_SNAPSHOT_DELETED) != null; + } + @Override public ReadOnlyList getChildrenList(int snapshotId) { return getParent().getChildrenList(snapshotId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java new file mode 100644 index 0000000000000..d57da2204aa38 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS; +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT; + +public class SnapshotDeletionGc { + public static final Logger LOG = LoggerFactory.getLogger( + SnapshotDeletionGc.class); + + private final FSNamesystem namesystem; + private final long deletionOrderedGcPeriodMs; + private final AtomicReference timer = new AtomicReference<>(); + + public SnapshotDeletionGc(FSNamesystem namesystem, Configuration conf) { + this.namesystem = namesystem; + + this.deletionOrderedGcPeriodMs = conf.getLong( + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS, + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT); + LOG.info("{} = {}", DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS, + deletionOrderedGcPeriodMs); + } + + public void schedule() { + if (timer.get() != null) { + return; + } + final Timer t = new Timer(getClass().getSimpleName(), true); + if (timer.compareAndSet(null, t)) { + LOG.info("Schedule at fixed rate {}", + StringUtils.formatTime(deletionOrderedGcPeriodMs)); + t.scheduleAtFixedRate(new GcTask(), + deletionOrderedGcPeriodMs, deletionOrderedGcPeriodMs); + } + } + + public void cancel() { + final Timer t = timer.getAndSet(null); + if (t != null) { + LOG.info("cancel"); + t.cancel(); + } + } + + private void gcDeletedSnapshot(String name) { + final Snapshot.Root deleted; + namesystem.readLock(); + try { + deleted = namesystem.getSnapshotManager().chooseDeletedSnapshot(); + } catch (Throwable e) { + LOG.error("Failed to chooseDeletedSnapshot", e); + throw e; + } finally { + namesystem.readUnlock(); + } + if (deleted == null) { + LOG.trace("{}: no snapshots are marked as deleted.", name); + return; + } + + final String snapshotRoot = deleted.getRootFullPathName(); + final String snapshotName = deleted.getLocalName(); + LOG.info("{}: delete snapshot {} from {}", + name, snapshotName, snapshotRoot); + + try { + namesystem.gcDeletedSnapshot(snapshotRoot, snapshotName); + } catch (Throwable e) { + LOG.error("Failed to gcDeletedSnapshot " + deleted.getFullPathName(), e); + } + } + + private class GcTask extends TimerTask { + private final AtomicInteger count = new AtomicInteger(); + + @Override + public void run() { + final int id = count.incrementAndGet(); + gcDeletedSnapshot(getClass().getSimpleName() + " #" + id); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index e9729abd42347..c908463d434f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import javax.management.ObjectName; @@ -84,6 +85,16 @@ public class SnapshotManager implements SnapshotStatsMXBean { public static final Logger LOG = LoggerFactory.getLogger(SnapshotManager.class); + // The following are private configurations + static final String DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + = "dfs.namenode.snapshot.deletion.ordered"; + static final boolean DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT + = false; + static final String DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS + = "dfs.namenode.snapshot.deletion.ordered.gc.period.ms"; + static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT + = 5 * 60_000L; //5 minutes + private final FSDirectory fsdir; private boolean captureOpenFiles; /** @@ -106,12 +117,13 @@ public class SnapshotManager implements SnapshotStatsMXBean { private static final int SNAPSHOT_ID_BIT_WIDTH = 28; private boolean allowNestedSnapshots = false; + private final boolean snapshotDeletionOrdered; private int snapshotCounter = 0; private final int maxSnapshotLimit; /** All snapshottable directories in the namesystem. */ private final Map snapshottables = - new HashMap(); + new ConcurrentHashMap<>(); public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { this.fsdir = fsdir; @@ -136,6 +148,12 @@ public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { + ", maxSnapshotLimit: " + maxSnapshotLimit); + this.snapshotDeletionOrdered = conf.getBoolean( + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_DEFAULT); + LOG.info("{} = {}", DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, + snapshotDeletionOrdered); + final int maxLevels = conf.getInt( DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_MAX_LEVELS, DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_MAX_SKIP_LEVELS_DEFAULT); @@ -145,6 +163,10 @@ public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { DirectoryDiffListFactory.init(skipInterval, maxLevels, LOG); } + public boolean isSnapshotDeletionOrdered() { + return snapshotDeletionOrdered; + } + @VisibleForTesting void setCaptureOpenFiles(boolean captureOpenFiles) { this.captureOpenFiles = captureOpenFiles; @@ -275,6 +297,49 @@ public INodeDirectory getSnapshottableRoot(final INodesInPath iip) return dir; } + public void assertMarkedAsDeleted(INodesInPath iip, String snapshotName) + throws IOException { + final INodeDirectory dir = getSnapshottableRoot(iip); + final Snapshot.Root snapshotRoot = dir.getDirectorySnapshottableFeature() + .getSnapshotByName(dir, snapshotName) + .getRoot(); + + if (!snapshotRoot.isMarkedAsDeleted()) { + throw new SnapshotException("Failed to gcDeletedSnapshot " + + snapshotName + " from " + dir.getFullPathName() + + ": snapshot is not marked as deleted"); + } + } + + void assertPrior(INodeDirectory dir, String snapshotName, int prior) + throws SnapshotException { + if (!isSnapshotDeletionOrdered()) { + return; + } + // prior must not exist + if (prior != Snapshot.NO_SNAPSHOT_ID) { + throw new SnapshotException("Failed to removeSnapshot " + + snapshotName + " from " + dir.getFullPathName() + + ": Unexpected prior (=" + prior + ") when " + + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + + " is " + isSnapshotDeletionOrdered()); + } + } + + void assertFirstSnapshot(INodeDirectory dir, + DirectorySnapshottableFeature snapshottable, Snapshot snapshot) + throws SnapshotException { + final INodeDirectoryAttributes first + = snapshottable.getDiffs().getFirstSnapshotINode(); + if (snapshot.getRoot() != first) { + throw new SnapshotException("Failed to delete snapshot " + snapshot + + " from " + dir.getFullPathName() + " since " + snapshot + + " is not the first snapshot (=" + first + ") and " + + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED + + " is " + isSnapshotDeletionOrdered()); + } + } + /** * Get the snapshot root directory for the given directory. The given * directory must either be a snapshot root or a descendant of any @@ -360,15 +425,14 @@ public String createSnapshot(final LeaseManager leaseManager, public void deleteSnapshot(final INodesInPath iip, final String snapshotName, INode.ReclaimContext reclaimContext, long now) throws IOException { final INodeDirectory srcRoot = getSnapshottableRoot(iip); - if (fsdir.isSnapshotDeletionOrdered()) { + if (isSnapshotDeletionOrdered()) { final DirectorySnapshottableFeature snapshottable = srcRoot.getDirectorySnapshottableFeature(); final Snapshot snapshot = snapshottable.getSnapshotByName( srcRoot, snapshotName); // Diffs must be not empty since a snapshot exists in the list - final int earliest = snapshottable.getDiffs().iterator().next() - .getSnapshotId(); + final int earliest = snapshottable.getDiffs().getFirst().getSnapshotId(); if (snapshot.getId() != earliest) { final XAttr snapshotXAttr = buildXAttr(); final List xattrs = Lists.newArrayListWithCapacity(1); @@ -390,8 +454,11 @@ public void deleteSnapshot(final INodesInPath iip, final String snapshotName, EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); return; } + + assertFirstSnapshot(srcRoot, snapshottable, snapshot); } - srcRoot.removeSnapshot(reclaimContext, snapshotName, now); + + srcRoot.removeSnapshot(reclaimContext, snapshotName, now, this); numSnapshots.getAndDecrement(); } @@ -435,9 +502,8 @@ void setSnapshotCounter(int counter) { snapshotCounter = counter; } - INodeDirectory[] getSnapshottableDirs() { - return snapshottables.values().toArray( - new INodeDirectory[snapshottables.size()]); + List getSnapshottableDirs() { + return new ArrayList<>(snapshottables.values()); } /** @@ -613,7 +679,7 @@ public int getMaxSnapshotID() { } public static XAttr buildXAttr() { - return XAttrHelper.buildXAttr(HdfsServerConstants.SNAPSHOT_XATTR_NAME); + return XAttrHelper.buildXAttr(HdfsServerConstants.XATTR_SNAPSHOT_DELETED); } private ObjectName mxBeanName; @@ -666,4 +732,35 @@ public static SnapshotInfo.Bean toBean(Snapshot s) { s.getRoot().getLocalName(), s.getRoot().getFullPathName(), s.getRoot().getModificationTime()); } -} + + private List getSnapshottableDirsForGc() { + final List dirs = getSnapshottableDirs(); + Collections.shuffle(dirs); + return dirs; + } + + Snapshot.Root chooseDeletedSnapshot() { + for(INodeDirectory dir : getSnapshottableDirsForGc()) { + final Snapshot.Root root = chooseDeletedSnapshot(dir); + if (root != null) { + return root; + } + } + return null; + } + + private static Snapshot.Root chooseDeletedSnapshot(INodeDirectory dir) { + final DirectorySnapshottableFeature snapshottable + = dir.getDirectorySnapshottableFeature(); + if (snapshottable == null) { + return null; + } + final DirectoryWithSnapshotFeature.DirectoryDiffList diffs + = snapshottable.getDiffs(); + final Snapshot.Root first = (Snapshot.Root)diffs.getFirstSnapshotINode(); + if (first == null || !first.isMarkedAsDeleted()) { + return null; + } + return first; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletion.java similarity index 76% rename from hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java rename to hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletion.java index d3097ea507770..efb72d1095b12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestOrderedSnapshotDeletion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletion.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs.server.namenode; +package org.apache.hadoop.hdfs.server.namenode.snapshot; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -26,9 +26,10 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -37,9 +38,8 @@ import java.util.EnumSet; import java.util.Map; -import static org.apache.hadoop.hdfs.DFSConfigKeys. - DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; -import static org.junit.Assert.assertNull; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SNAPSHOT_DELETED; +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; import static org.junit.Assert.assertTrue; /** @@ -71,7 +71,7 @@ public void tearDown() throws Exception { } @Test(timeout = 60000) - public void testConf() throws Exception { + public void testOrderedSnapshotDeletion() throws Exception { DistributedFileSystem hdfs = cluster.getFileSystem(); hdfs.mkdirs(snapshottableDir); hdfs.allowSnapshot(snapshottableDir); @@ -96,22 +96,54 @@ public void testConf() throws Exception { hdfs.deleteSnapshot(snapshottableDir, "s2"); } + static void assertMarkedAsDeleted(Path snapshotRoot, MiniDFSCluster cluster) + throws IOException { + // Check if the path exists + Assert.assertNotNull(cluster.getFileSystem().getFileStatus(snapshotRoot)); + + // Check xAttr for snapshotRoot + final INode inode = cluster.getNamesystem().getFSDirectory() + .getINode(snapshotRoot.toString()); + final XAttrFeature f = inode.getXAttrFeature(); + final XAttr xAttr = f.getXAttr(XATTR_SNAPSHOT_DELETED); + Assert.assertNotNull(xAttr); + Assert.assertEquals(XATTR_SNAPSHOT_DELETED.substring("system.".length()), + xAttr.getName()); + Assert.assertEquals(XAttr.NameSpace.SYSTEM, xAttr.getNameSpace()); + Assert.assertNull(xAttr.getValue()); + + // Check inode + Assert.assertTrue(inode instanceof Snapshot.Root); + Assert.assertTrue(((Snapshot.Root)inode).isMarkedAsDeleted()); + } + + static void assertNotMarkedAsDeleted(Path snapshotRoot, + MiniDFSCluster cluster) throws IOException { + // Check if the path exists + Assert.assertNotNull(cluster.getFileSystem().getFileStatus(snapshotRoot)); + + // Check xAttr for snapshotRoot + final INode inode = cluster.getNamesystem().getFSDirectory() + .getINode(snapshotRoot.toString()); + final XAttrFeature f = inode.getXAttrFeature(); + if (f != null) { + final XAttr xAttr = f.getXAttr(XATTR_SNAPSHOT_DELETED); + Assert.assertNull(xAttr); + } + + // Check inode + Assert.assertTrue(inode instanceof Snapshot.Root); + Assert.assertFalse(((Snapshot.Root)inode).isMarkedAsDeleted()); + } + void assertXAttrSet(String snapshot, DistributedFileSystem hdfs, XAttr newXattr) throws IOException { hdfs.deleteSnapshot(snapshottableDir, snapshot); // Check xAttr for parent directory - FSNamesystem namesystem = cluster.getNamesystem(); Path snapshotRoot = SnapshotTestHelper.getSnapshotRoot(snapshottableDir, snapshot); - INode inode = namesystem.getFSDirectory().getINode(snapshotRoot.toString()); - XAttrFeature f = inode.getXAttrFeature(); - XAttr xAttr = f.getXAttr(HdfsServerConstants.SNAPSHOT_XATTR_NAME); - assertTrue("Snapshot xAttr should exist", xAttr != null); - assertTrue(xAttr.getName().equals(HdfsServerConstants.SNAPSHOT_XATTR_NAME. - replace("system.", ""))); - assertTrue(xAttr.getNameSpace().equals(XAttr.NameSpace.SYSTEM)); - assertNull(xAttr.getValue()); + assertMarkedAsDeleted(snapshotRoot, cluster); // Make sure its not user visible if (cluster.getNameNode().getConf().getBoolean(DFSConfigKeys. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java new file mode 100644 index 0000000000000..b35134ba7b051 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.event.Level; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS; +import static org.apache.hadoop.hdfs.server.namenode.snapshot.TestOrderedSnapshotDeletion.assertMarkedAsDeleted; +import static org.apache.hadoop.hdfs.server.namenode.snapshot.TestOrderedSnapshotDeletion.assertNotMarkedAsDeleted; + +/** + * Test {@link SnapshotDeletionGc}. + */ +public class TestOrderedSnapshotDeletionGc { + private static final int GC_PERIOD = 10; + + private MiniDFSCluster cluster; + + @Before + public void setUp() throws Exception { + final Configuration conf = new Configuration(); + conf.setBoolean(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, true); + conf.setInt(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS, GC_PERIOD); + + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster.waitActive(); + + GenericTestUtils.setLogLevel(SnapshotDeletionGc.LOG, Level.TRACE); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Test(timeout = 60000) + public void testSingleDir() throws Exception { + final DistributedFileSystem hdfs = cluster.getFileSystem(); + + final Path snapshottableDir = new Path("/dir"); + hdfs.mkdirs(snapshottableDir); + hdfs.allowSnapshot(snapshottableDir); + + final Path sub0 = new Path(snapshottableDir, "sub0"); + hdfs.mkdirs(sub0); + final Path s0path = hdfs.createSnapshot(snapshottableDir, "s0"); + Assert.assertTrue(exist(s0path, hdfs)); + + final Path sub1 = new Path(snapshottableDir, "sub1"); + hdfs.mkdirs(sub1); + final Path s1path = hdfs.createSnapshot(snapshottableDir, "s1"); + Assert.assertTrue(exist(s1path, hdfs)); + + final Path sub2 = new Path(snapshottableDir, "sub2"); + hdfs.mkdirs(sub2); + final Path s2path = hdfs.createSnapshot(snapshottableDir, "s2"); + Assert.assertTrue(exist(s2path, hdfs)); + + assertNotMarkedAsDeleted(s0path, cluster); + assertNotMarkedAsDeleted(s1path, cluster); + assertNotMarkedAsDeleted(s2path, cluster); + + hdfs.deleteSnapshot(snapshottableDir, "s2"); + assertNotMarkedAsDeleted(s0path, cluster); + assertNotMarkedAsDeleted(s1path, cluster); + assertMarkedAsDeleted(s2path, cluster); + + hdfs.deleteSnapshot(snapshottableDir, "s1"); + assertNotMarkedAsDeleted(s0path, cluster); + assertMarkedAsDeleted(s1path, cluster); + assertMarkedAsDeleted(s2path, cluster); + + // should not be gc'ed + Thread.sleep(10*GC_PERIOD); + assertNotMarkedAsDeleted(s0path, cluster); + assertMarkedAsDeleted(s1path, cluster); + assertMarkedAsDeleted(s2path, cluster); + + hdfs.deleteSnapshot(snapshottableDir, "s0"); + Assert.assertFalse(exist(s0path, hdfs)); + + waitForGc(Arrays.asList(s1path, s2path), hdfs); + } + + static boolean exist(Path snapshotRoot, DistributedFileSystem hdfs) + throws IOException { + try { + hdfs.getFileStatus(snapshotRoot); + return true; + } catch (FileNotFoundException ignored) { + return false; + } + } + + static void waitForGc(List snapshotPaths, DistributedFileSystem hdfs) + throws Exception { + final Iterator i = snapshotPaths.iterator(); + for(Path p = i.next();; Thread.sleep(GC_PERIOD)) { + for(; !exist(p, hdfs); p = i.next()) { + if (!i.hasNext()) { + return; + } + } + } + } + + @Test(timeout = 60000) + public void testMultipleDirs() throws Exception { + final int numSnapshottables = 10; + final DistributedFileSystem hdfs = cluster.getFileSystem(); + + final List snapshottableDirs = new ArrayList<>(); + for(int i = 0; i < numSnapshottables; i++) { + final Path p = new Path("/dir" + i); + snapshottableDirs.add(p); + hdfs.mkdirs(p); + hdfs.allowSnapshot(p); + } + + final Random random = new Random(); + final List snapshotPaths = new ArrayList<>(); + for(Path s : snapshottableDirs) { + final int numSnapshots = random.nextInt(10) + 1; + createSnapshots(s, numSnapshots, snapshotPaths, hdfs); + } + + // Randomly delete snapshots + Collections.shuffle(snapshotPaths); + for(Path p : snapshotPaths) { + hdfs.deleteSnapshot(p.getParent().getParent(), p.getName()); + } + + waitForGc(snapshotPaths, hdfs); + } + + static void createSnapshots(Path snapshottableDir, int numSnapshots, + List snapshotPaths, DistributedFileSystem hdfs) + throws IOException { + for(int i = 0; i < numSnapshots; i++) { + final Path sub = new Path(snapshottableDir, "sub" + i); + hdfs.mkdirs(sub); + final Path p = hdfs.createSnapshot(snapshottableDir, "s" + i); + snapshotPaths.add(p); + Assert.assertTrue(exist(p, hdfs)); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java index 3e318b3a30505..b85530c3cbee6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java @@ -156,7 +156,7 @@ public void testApplyEditLogForDeletion() throws Exception { assertEquals(2, cluster.getNamesystem().getSnapshotManager() .getNumSnapshottableDirs()); assertEquals(2, cluster.getNamesystem().getSnapshotManager() - .getSnapshottableDirs().length); + .getSnapshottableDirs().size()); // delete /foo hdfs.delete(foo, true); @@ -165,7 +165,7 @@ public void testApplyEditLogForDeletion() throws Exception { assertEquals(0, cluster.getNamesystem().getSnapshotManager() .getNumSnapshottableDirs()); assertEquals(0, cluster.getNamesystem().getSnapshotManager() - .getSnapshottableDirs().length); + .getSnapshottableDirs().size()); hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); hdfs.saveNamespace(); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); From e756fe3590906bfd8ffe4ab5cc8b9b24a9b2b4b2 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 31 Jul 2020 15:49:49 +0800 Subject: [PATCH 027/335] HDFS-14950. fix missing libhdfspp lib in dist-package (#1947) libhdfspp.{a,so} are missed in dist-package. This patch fixed this by copying these libs to the right directory Signed-off-by: Yuan Zhou --- .../src/main/native/libhdfspp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 6a2f378d0a4bb..939747c34b444 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -265,6 +265,7 @@ if (HADOOP_BUILD) ${CMAKE_THREAD_LIBS_INIT} ) set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION}) + hadoop_dual_output_directory(hdfspp ${OUT_DIR}) else (HADOOP_BUILD) add_library(hdfspp_static STATIC ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) target_link_libraries(hdfspp_static From a7fda2e38f2a06e18c2929dff0be978d5e0ef9d5 Mon Sep 17 00:00:00 2001 From: bilaharith <52483117+bilaharith@users.noreply.github.com> Date: Sat, 1 Aug 2020 00:57:57 +0530 Subject: [PATCH 028/335] HADOOP-17137. ABFS: Makes the test cases in ITestAbfsNetworkStatistics agnostic - Contributed by Bilahari T H --- .../azurebfs/ITestAbfsNetworkStatistics.java | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java index b2e1301152111..9d76fb00229b3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java @@ -32,6 +32,9 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CONNECTIONS_MADE; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.SEND_REQUESTS; + public class ITestAbfsNetworkStatistics extends AbstractAbfsIntegrationTest { private static final Logger LOG = @@ -56,6 +59,11 @@ public void testAbfsHttpSendStatistics() throws IOException { String testNetworkStatsString = "http_send"; long connectionsMade, requestsSent, bytesSent; + metricMap = fs.getInstrumentationMap(); + long connectionsMadeBeforeTest = metricMap + .get(CONNECTIONS_MADE.getStatName()); + long requestsMadeBeforeTest = metricMap.get(SEND_REQUESTS.getStatName()); + /* * Creating AbfsOutputStream will result in 1 connection made and 1 send * request. @@ -75,27 +83,26 @@ public void testAbfsHttpSendStatistics() throws IOException { /* * Testing the network stats with 1 write operation. * - * connections_made : 3(getFileSystem()) + 1(AbfsOutputStream) + 2(flush). + * connections_made : (connections made above) + 2(flush). * - * send_requests : 1(getFileSystem()) + 1(AbfsOutputStream) + 2(flush). + * send_requests : (requests sent above) + 2(flush). * * bytes_sent : bytes wrote in AbfsOutputStream. */ - if (fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(sendRequestPath).toString())) { + long extraCalls = 0; + if (!fs.getAbfsStore() + .isAppendBlobKey(fs.makeQualified(sendRequestPath).toString())) { // no network calls are made for hflush in case of appendblob - connectionsMade = assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE, - 5, metricMap); - requestsSent = assertAbfsStatistics(AbfsStatistic.SEND_REQUESTS, 3, - metricMap); - } else { - connectionsMade = assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE, - 6, metricMap); - requestsSent = assertAbfsStatistics(AbfsStatistic.SEND_REQUESTS, 4, - metricMap); + extraCalls++; } + long expectedConnectionsMade = connectionsMadeBeforeTest + extraCalls + 2; + long expectedRequestsSent = requestsMadeBeforeTest + extraCalls + 2; + connectionsMade = assertAbfsStatistics(CONNECTIONS_MADE, + expectedConnectionsMade, metricMap); + requestsSent = assertAbfsStatistics(SEND_REQUESTS, expectedRequestsSent, + metricMap); bytesSent = assertAbfsStatistics(AbfsStatistic.BYTES_SENT, testNetworkStatsString.getBytes().length, metricMap); - } // To close the AbfsOutputStream 1 connection is made and 1 request is sent. @@ -135,14 +142,14 @@ public void testAbfsHttpSendStatistics() throws IOException { */ if (fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(sendRequestPath).toString())) { // no network calls are made for hflush in case of appendblob - assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE, + assertAbfsStatistics(CONNECTIONS_MADE, connectionsMade + 1 + LARGE_OPERATIONS, metricMap); - assertAbfsStatistics(AbfsStatistic.SEND_REQUESTS, + assertAbfsStatistics(SEND_REQUESTS, requestsSent + 1 + LARGE_OPERATIONS, metricMap); } else { - assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE, + assertAbfsStatistics(CONNECTIONS_MADE, connectionsMade + 1 + LARGE_OPERATIONS * 2, metricMap); - assertAbfsStatistics(AbfsStatistic.SEND_REQUESTS, + assertAbfsStatistics(SEND_REQUESTS, requestsSent + 1 + LARGE_OPERATIONS * 2, metricMap); } assertAbfsStatistics(AbfsStatistic.BYTES_SENT, @@ -182,6 +189,10 @@ public void testAbfsHttpResponseStatistics() throws IOException { out.write(testResponseString.getBytes()); out.hflush(); + metricMap = fs.getInstrumentationMap(); + long getResponsesBeforeTest = metricMap + .get(CONNECTIONS_MADE.getStatName()); + // open would require 1 get response. in = fs.open(getResponsePath); // read would require 1 get response and also get the bytes received. @@ -195,18 +206,20 @@ public void testAbfsHttpResponseStatistics() throws IOException { /* * Testing values of statistics after writing and reading a buffer. * - * get_responses - 6(above operations) + 1(open()) + 1 (read()). + * get_responses - (above operations) + 1(open()) + 1 (read()).; * * bytes_received - This should be equal to bytes sent earlier. */ - if (fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(getResponsePath).toString())) { - //for appendBlob hflush is a no-op - getResponses = assertAbfsStatistics(AbfsStatistic.GET_RESPONSES, 7, - metricMap); - } else { - getResponses = assertAbfsStatistics(AbfsStatistic.GET_RESPONSES, 8, - metricMap); + long extraCalls = 0; + if (!fs.getAbfsStore() + .isAppendBlobKey(fs.makeQualified(getResponsePath).toString())) { + // no network calls are made for hflush in case of appendblob + extraCalls++; } + long expectedGetResponses = getResponsesBeforeTest + extraCalls + 1; + getResponses = assertAbfsStatistics(AbfsStatistic.GET_RESPONSES, + expectedGetResponses, metricMap); + // Testing that bytes received is equal to bytes sent. long bytesSend = metricMap.get(AbfsStatistic.BYTES_SENT.getStatName()); bytesReceived = assertAbfsStatistics(AbfsStatistic.BYTES_RECEIVED, From 5323e83edfe63355ec38ffdaacc0c27d14cad31c Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Sat, 1 Aug 2020 13:03:46 +0530 Subject: [PATCH 029/335] YARN-10359. Log container report only if list is not empty. Contributed by Bilwa S T. --- .../hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 37da31a322b3e..14b360ca98da7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -398,7 +398,7 @@ protected void registerWithRM() nodeManagerVersionId, containerReports, getRunningApplications(), nodeLabels, physicalResource, nodeAttributes, nodeStatus); - if (containerReports != null) { + if (containerReports != null && !containerReports.isEmpty()) { LOG.info("Registering with RM using containers :" + containerReports); } if (logAggregationEnabled) { From 528a799a784c95cc702e3547a8a294f00743533b Mon Sep 17 00:00:00 2001 From: hemanthboyina Date: Sun, 2 Aug 2020 12:02:37 +0530 Subject: [PATCH 030/335] HDFS-15229. Truncate info should be logged at INFO level. Contributed by Ravuri Sushma sree. --- .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +- .../hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 72ece88b92a9b..c0cbabb8401fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2270,7 +2270,7 @@ boolean truncate(String src, long newLength, String clientName, requireEffectiveLayoutVersionForFeature(Feature.TRUNCATE); FSDirTruncateOp.TruncateResult r = null; try { - NameNode.stateChangeLog.debug( + NameNode.stateChangeLog.info( "DIR* NameSystem.truncate: src={} newLength={}", src, newLength); if (newLength < 0) { throw new HadoopIllegalArgumentException( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index b42252d9aa9d4..3f1fb5a93915a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -1100,10 +1100,8 @@ public void rename2(String src, String dst, Options.Rename... options) public boolean truncate(String src, long newLength, String clientName) throws IOException { checkNNStartup(); - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* NameNode.truncate: " + src + " to " + - newLength); - } + stateChangeLog + .debug("*DIR* NameNode.truncate: " + src + " to " + newLength); String clientMachine = getClientMachine(); try { return namesystem.truncate( From d8a2df25ad2a145712910259f4d4079c302c2aef Mon Sep 17 00:00:00 2001 From: bshashikant Date: Sun, 2 Aug 2020 21:59:56 +0530 Subject: [PATCH 031/335] HDFS-15498. Show snapshots deletion status in snapList cmd. (#2181) --- .../hadoop/hdfs/protocol/SnapshotStatus.java | 37 ++++++++++++++----- .../hdfs/protocolPB/PBHelperClient.java | 2 + .../src/main/proto/hdfs.proto | 1 + .../namenode/snapshot/SnapshotManager.java | 3 +- .../namenode/snapshot/TestListSnapshot.java | 14 ++++++- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java index 6a938a9e4b17b..8c0dabd34ad42 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java @@ -40,6 +40,11 @@ public class SnapshotStatus { */ private final int snapshotID; + /** + * Whether the snapshot is deleted or not. + */ + private final boolean isDeleted; + /** * Full path of the parent. */ @@ -50,7 +55,7 @@ public SnapshotStatus(long modificationTime, long accessTime, EnumSet flags, String owner, String group, byte[] localName, long inodeId, int childrenNum, int snapshotID, - byte[] parentFullPath) { + boolean isDeleted, byte[] parentFullPath) { this.dirStatus = new HdfsFileStatus.Builder() .isdir(true) .mtime(modificationTime) @@ -64,13 +69,7 @@ public SnapshotStatus(long modificationTime, long accessTime, .children(childrenNum) .build(); this.snapshotID = snapshotID; - this.parentFullPath = parentFullPath; - } - - public SnapshotStatus(HdfsFileStatus dirStatus, - int snapshotNumber, byte[] parentFullPath) { - this.dirStatus = dirStatus; - this.snapshotID = snapshotNumber; + this.isDeleted = isDeleted; this.parentFullPath = parentFullPath; } @@ -89,6 +88,13 @@ public int getSnapshotID() { return snapshotID; } + /** + * @return whether snapshot is deleted + */ + public boolean isDeleted() { + return isDeleted; + } + /** * @return The basic information of the directory */ @@ -143,6 +149,7 @@ public static void print(SnapshotStatus[] stats, + "%" + maxLen + "s " + "%s " // mod time + "%" + maxSnapshotID + "s " + + "%s " // deletion status + "%s"; // path SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); @@ -155,6 +162,7 @@ public static void print(SnapshotStatus[] stats, String.valueOf(status.dirStatus.getLen()), dateFormat.format(new Date(status.dirStatus.getModificationTime())), status.snapshotID, + status.isDeleted ? "DELETED" : "ACTIVE", getSnapshotPath(DFSUtilClient.bytes2String(status.parentFullPath), status.dirStatus.getLocalName()) ); @@ -166,6 +174,9 @@ private static int maxLength(int n, Object value) { return Math.max(n, String.valueOf(value).length()); } + /** + * To be used to for collection of snapshot jmx. + */ public static class Bean { private final String path; private final int snapshotID; @@ -173,15 +184,19 @@ public static class Bean { private final short permission; private final String owner; private final String group; + private final boolean isDeleted; + public Bean(String path, int snapshotID, long - modificationTime, short permission, String owner, String group) { + modificationTime, short permission, String owner, String group, + boolean isDeleted) { this.path = path; this.snapshotID = snapshotID; this.modificationTime = modificationTime; this.permission = permission; this.owner = owner; this.group = group; + this.isDeleted = isDeleted; } public String getPath() { @@ -207,6 +222,10 @@ public String getOwner() { public String getGroup() { return group; } + + public boolean isDeleted() { + return isDeleted; + } } static String getSnapshotPath(String snapshottableDir, diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index efe020a3acf9c..f3d0cd96ed129 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1711,6 +1711,7 @@ public static SnapshotStatus convert( status.getFileId(), status.getChildrenNum(), sdirStatusProto.getSnapshotID(), + sdirStatusProto.getIsDeleted(), sdirStatusProto.getParentFullpath().toByteArray()); } @@ -2425,6 +2426,7 @@ public static HdfsProtos.SnapshotStatusProto convert(SnapshotStatus status) { .newBuilder() .setSnapshotID(status.getSnapshotID()) .setParentFullpath(parentFullPathBytes) + .setIsDeleted(status.isDeleted()) .setDirStatus(fs); return builder.build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 3e24d73ce2d26..2440a7ac16d9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -574,6 +574,7 @@ message SnapshotStatusProto { // Fields specific for snapshot directory required uint32 snapshotID = 2; required bytes parent_fullpath = 3; + required bool isDeleted = 4; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index c908463d434f8..d56611235a9cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -599,7 +599,8 @@ public SnapshotStatus[] getSnapshotListing(INodesInPath iip) // It is expensive to build the snapshot tree for the directory // and determine the child count. dir.getChildrenNum(Snapshot.CURRENT_STATE_ID), - s.getId(), DFSUtil.string2Bytes(dir.getParent().getFullPathName())); + s.getId(), s.getRoot().isMarkedAsDeleted(), + DFSUtil.string2Bytes(dir.getParent().getFullPathName())); } return statuses; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java index dcff2f7f7c69c..672f21aed370c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestListSnapshot.java @@ -30,7 +30,10 @@ import org.junit.Before; import org.junit.Test; +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager. + DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -51,6 +54,7 @@ public class TestListSnapshot { @Before public void setUp() throws Exception { conf = new Configuration(); + conf.setBoolean(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, true); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) .build(); cluster.waitActive(); @@ -128,7 +132,15 @@ public void testListSnapshot() throws Exception { snapshotStatuses[2].getFullPath()); hdfs.deleteSnapshot(dir1, "s2"); snapshotStatuses = hdfs.getSnapshotListing(dir1); - // There are now 2 snapshots for dir1 + // There are now 2 active snapshots for dir1 and one is marked deleted + assertEquals(3, snapshotStatuses.length); + assertTrue(snapshotStatuses[2].isDeleted()); + assertFalse(snapshotStatuses[1].isDeleted()); + assertFalse(snapshotStatuses[0].isDeleted()); + // delete the 1st snapshot + hdfs.deleteSnapshot(dir1, "s0"); + snapshotStatuses = hdfs.getSnapshotListing(dir1); + // There are now 2 snapshots now as the 1st one is deleted in order assertEquals(2, snapshotStatuses.length); } } \ No newline at end of file From c40cbc57fa20d385a971a4b07af13fa28d5908c9 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 3 Aug 2020 10:46:51 +0900 Subject: [PATCH 032/335] HADOOP-17091. [JDK11] Fix Javadoc errors (#2098) --- Jenkinsfile | 5 ++++- hadoop-common-project/hadoop-common/pom.xml | 1 + hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 + hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 1 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + hadoop-project/pom.xml | 3 +++ hadoop-tools/hadoop-aws/pom.xml | 1 + 7 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0461c5727aff9..7b5b15482f226 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ pipeline { DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" YETUS='yetus' // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' - YETUS_VERSION='rel/0.12.0' + YETUS_VERSION='master' } parameters { @@ -159,6 +159,9 @@ pipeline { YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") YETUS_ARGS+=("--multijdktests=compile") + # custom javadoc goals + YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork") + "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ''' } diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 9bb70ac76a06a..fb4193a074a33 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -35,6 +35,7 @@ true ../etc/hadoop wsce-site.xml + true diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index d65e6030369b3..c4bc07f29c431 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -31,6 +31,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdfs + true diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 782f28a0ef30e..489458ca97352 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -31,6 +31,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdfs + true diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index e1ea5840deda5..5b50062a31666 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -32,6 +32,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdfs true + true diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7b7aab96ba00d..06fcee381d116 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -152,6 +152,7 @@ 1.8 + false -html4 diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 2383da9c104c7..5cdaf26007f7a 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -35,6 +35,7 @@ UTF-8 true ${project.build.directory}/test + true unset From eac558380fd7d3c2e78b8956e2080688bb1dd8bb Mon Sep 17 00:00:00 2001 From: Brahma Reddy Battula Date: Mon, 3 Aug 2020 12:54:36 +0530 Subject: [PATCH 033/335] YARN-10229. [Federation] Client should be able to submit application to RM directly using normal client conf. Contributed by Bilwa S T. --- .../amrmproxy/AMRMProxyService.java | 35 +++++++++++++++++-- .../amrmproxy/TestAMRMProxyService.java | 21 +++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java index d3c4a1d528861..fe278f345397f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java @@ -108,6 +108,8 @@ public class AMRMProxyService extends CompositeService implements private Map applPipelineMap; private RegistryOperations registry; private AMRMProxyMetrics metrics; + private FederationStateStoreFacade federationFacade; + private boolean federationEnabled = false; /** * Creates an instance of the service. @@ -144,7 +146,10 @@ protected void serviceInit(Configuration conf) throws Exception { RegistryOperations.class); addService(this.registry); } - + this.federationFacade = FederationStateStoreFacade.getInstance(); + this.federationEnabled = + conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_ENABLED); super.serviceInit(conf); } @@ -389,13 +394,22 @@ public void processApplicationStartRequest(StartContainerRequest request) throws IOException, YarnException { long startTime = clock.getTime(); try { - LOG.info("Callback received for initializing request " - + "processing pipeline for an AM"); ContainerTokenIdentifier containerTokenIdentifierForKey = BuilderUtils.newContainerTokenIdentifier(request.getContainerToken()); ApplicationAttemptId appAttemptId = containerTokenIdentifierForKey.getContainerID() .getApplicationAttemptId(); + ApplicationId applicationID = appAttemptId.getApplicationId(); + // Checking if application is there in federation state store only + // if federation is enabled. If + // application is submitted to router then it adds it in statestore. + // if application is not found in statestore that means its + // submitted to RM + if (!checkIfAppExistsInStateStore(applicationID)) { + return; + } + LOG.info("Callback received for initializing request " + + "processing pipeline for an AM"); Credentials credentials = YarnServerSecurityUtils .parseCredentials(request.getContainerLaunchContext()); @@ -772,6 +786,21 @@ private RequestInterceptorChainWrapper getInterceptorChain( } } + boolean checkIfAppExistsInStateStore(ApplicationId applicationID) { + if (!federationEnabled) { + return true; + } + + try { + // Check if app is there in state store. If app is not there then it + // throws Exception + this.federationFacade.getApplicationHomeSubCluster(applicationID); + } catch (YarnException ex) { + return false; + } + return true; + } + @SuppressWarnings("unchecked") private Token getFirstAMRMToken( Collection> allTokens) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java index b269fa45677ee..60e383870fe28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java @@ -662,6 +662,27 @@ public void testAppRecoveryFailure() throws YarnException, Exception { Assert.assertEquals(0, state.getAppContexts().size()); } + @Test + public void testCheckIfAppExistsInStateStore() + throws IOException, YarnException { + ApplicationId appId = ApplicationId.newInstance(0, 0); + Configuration conf = createConfiguration(); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + + createAndStartAMRMProxyService(conf); + + Assert.assertEquals(false, + getAMRMProxyService().checkIfAppExistsInStateStore(appId)); + + Configuration distConf = createConfiguration(); + conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true); + + createAndStartAMRMProxyService(distConf); + + Assert.assertEquals(true, + getAMRMProxyService().checkIfAppExistsInStateStore(appId)); + } + /** * A mock intercepter implementation that uses the same mockRM instance across * restart. From 82f3ffcd64d25cf3a2f5e280e07140994e0ba8cb Mon Sep 17 00:00:00 2001 From: hemanthboyina Date: Mon, 3 Aug 2020 23:21:01 +0530 Subject: [PATCH 034/335] HDFS-15503. File and directory permissions are not able to be modified from WebUI. --- .../hadoop-hdfs-rbf/src/main/webapps/router/explorer.js | 4 +++- .../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js index 490c3934adb6d..cb16eac7b1221 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js @@ -108,7 +108,8 @@ */ function view_perm_details(e, filename, abs_path, perms) { $('.explorer-perm-links').popover('destroy'); - e.popover({html: true, content: $('#explorer-popover-perm-info').html(), trigger: 'focus'}) + setTimeout(function() { + e.popover({html: true,sanitize: false, content: $('#explorer-popover-perm-info').html(), trigger: 'focus'}) .on('shown.bs.popover', function(e) { var popover = $(this), parent = popover.parent(); //Convert octal to binary permissions @@ -122,6 +123,7 @@ }); }) .popover('show'); + }, 100); } // Use WebHDFS to set permissions on an absolute path diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js index cbf9df99567c4..ea8b0accbde6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js @@ -108,7 +108,8 @@ */ function view_perm_details(e, filename, abs_path, perms) { $('.explorer-perm-links').popover('destroy'); - e.popover({html: true, content: $('#explorer-popover-perm-info').html(), trigger: 'focus'}) + setTimeout(function() { + e.popover({html: true,sanitize: false, content: $('#explorer-popover-perm-info').html(), trigger: 'focus'}) .on('shown.bs.popover', function(e) { var popover = $(this), parent = popover.parent(); //Convert octal to binary permissions @@ -122,6 +123,7 @@ }); }) .popover('show'); + }, 100); } // Use WebHDFS to set permissions on an absolute path From ab2b3df2de1d7c490a4aa83460328bcd910c39d4 Mon Sep 17 00:00:00 2001 From: kevinzhao1661 Date: Tue, 4 Aug 2020 11:02:16 +0800 Subject: [PATCH 035/335] YARN-10383. YarnCommands.md is inconsistent with the source code (#2177) --- .../hadoop-yarn-site/src/site/markdown/YarnCommands.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 0f2b379f98902..c1d939b107351 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -214,7 +214,7 @@ Usage: -getGroups [username] -addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3"> -removeFromClusterNodeLabels (label splitted by ",") - -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> [-failOnUnknownNodes] + -replaceLabelsOnNode <"node1[:port]=label1 node2[:port]=label2"> [-failOnUnknownNodes] -directlyAccessNodeLabelStore -refreshClusterMaxPriority -updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout]) or -updateNodeResource [NodeID] [ResourceTypes] ([OvercommitTimeout]) @@ -238,7 +238,7 @@ Usage: | -getGroups [username] | Get groups the specified user belongs to. | | -addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3"> | Add to cluster node labels. Default exclusivity is true. | | -removeFromClusterNodeLabels (label splitted by ",") | Remove from cluster node labels. | -| -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> [-failOnUnknownNodes]| Replace labels on nodes (please note that we do not support specifying multiple labels on a single host for now.) -failOnUnknownNodes is optional, when we set this option, it will fail if specified nodes are unknown.| +| -replaceLabelsOnNode <"node1[:port]=label1 node2[:port]=label2"> [-failOnUnknownNodes]| Replace labels on nodes (please note that we do not support specifying multiple labels on a single host for now.) -failOnUnknownNodes is optional, when we set this option, it will fail if specified nodes are unknown.| | -directlyAccessNodeLabelStore | This is DEPRECATED, will be removed in future releases. Directly access node label store, with this option, all node label related operations will not connect RM. Instead, they will access/modify stored node labels directly. By default, it is false (access via RM). AND PLEASE NOTE: if you configured yarn.node-labels.fs-store.root-dir to a local directory (instead of NFS or HDFS), this option will only work when the command run on the machine where RM is running. | | -refreshClusterMaxPriority | Refresh cluster max priority | | -updateNodeResource [NodeID] [MemSize] [vCores] \([OvercommitTimeout]\) | Update resource on specific node. | From e072d33327b8f5d38b74a15e279d492ad379a47c Mon Sep 17 00:00:00 2001 From: bshashikant Date: Tue, 4 Aug 2020 14:10:29 +0530 Subject: [PATCH 036/335] HDFS-15497. Make snapshot limit on global as well per snapshot root directory configurable (#2175) --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +++ .../namenode/snapshot/SnapshotManager.java | 30 +++++++++-- .../src/main/resources/hdfs-default.xml | 9 ++++ .../snapshot/TestSnapshotManager.java | 53 ++++++++++++++----- 4 files changed, 82 insertions(+), 16 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 3865f9b4d72c5..d1c32f122eca5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -501,6 +501,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_SNAPSHOT_MAX_LIMIT = "dfs.namenode.snapshot.max.limit"; public static final int DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT = 65536; + public static final String + DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT = + "dfs.namenode.snapshot.filesystem.limit"; + // default value is same as snapshot quota set for a snapshottable directory + public static final int + DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT_DEFAULT = 65536; public static final String DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL = "dfs.namenode.snapshot.skiplist.interval"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index d56611235a9cb..7569fc64e6666 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -120,12 +120,14 @@ public class SnapshotManager implements SnapshotStatsMXBean { private final boolean snapshotDeletionOrdered; private int snapshotCounter = 0; private final int maxSnapshotLimit; + private final int maxSnapshotFSLimit; /** All snapshottable directories in the namesystem. */ private final Map snapshottables = new ConcurrentHashMap<>(); - public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { + public SnapshotManager(final Configuration conf, final FSDirectory fsdir) + throws SnapshotException { this.fsdir = fsdir; this.captureOpenFiles = conf.getBoolean( DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES, @@ -138,13 +140,20 @@ public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { DFSConfigKeys. DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT_DEFAULT); this.maxSnapshotLimit = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_MAX_LIMIT, - DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT); + DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_MAX_LIMIT, + DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_MAX_LIMIT_DEFAULT); + this.maxSnapshotFSLimit = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT, + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT_DEFAULT); LOG.info("Loaded config captureOpenFiles: " + captureOpenFiles + ", skipCaptureAccessTimeOnlyChange: " + skipCaptureAccessTimeOnlyChange + ", snapshotDiffAllowSnapRootDescendant: " + snapshotDiffAllowSnapRootDescendant + + ", maxSnapshotFSLimit: " + + maxSnapshotFSLimit + ", maxSnapshotLimit: " + maxSnapshotLimit); @@ -160,6 +169,13 @@ public SnapshotManager(final Configuration conf, final FSDirectory fsdir) { final int skipInterval = conf.getInt( DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL, DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIPLIST_SKIP_INTERVAL_DEFAULT); + if (maxSnapshotLimit > maxSnapshotFSLimit) { + final String errMsg = DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_MAX_LIMIT + + " cannot be greater than " + + DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT; + throw new SnapshotException(errMsg); + } DirectoryDiffListFactory.init(skipInterval, maxLevels, LOG); } @@ -405,6 +421,14 @@ public String createSnapshot(final LeaseManager leaseManager, "Failed to create the snapshot. The FileSystem has run out of " + "snapshot IDs and ID rollover is not supported."); } + int n = numSnapshots.get(); + if (n >= maxSnapshotFSLimit) { + // We have reached the maximum snapshot limit + throw new SnapshotException( + "Failed to create snapshot: there are already " + (n + 1) + + " snapshot(s) and the max snapshot limit is " + + maxSnapshotFSLimit); + } srcRoot.addSnapshot(snapshotCounter, snapshotName, leaseManager, this.captureOpenFiles, maxSnapshotLimit, mtime); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 689ecfe2f3342..db152e60bd65a 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5101,6 +5101,15 @@ for maximum no of snapshots allowed is 65536. + + dfs.namenode.snapshot.filesystem.limit + 65536 + + Limits the maximum number of snapshots allowed on the entire filesystem. + If the configuration is not set, the default limit + for maximum no of snapshots allowed is 65536. + + dfs.namenode.snapshot.skiplist.max.levels diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java index 4d6e6f4172c02..b1e91686b3d57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.spy; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.INode; @@ -35,32 +36,54 @@ import org.junit.Assert; import org.junit.Test; +import java.io.IOException; + /** * Testing snapshot manager functionality. */ public class TestSnapshotManager { - private static final int testMaxSnapshotLimit = 7; + private static final int testMaxSnapshotIDLimit = 7; /** - * Test that the global limit on snapshots is honored. + * Test that the global limit on snapshot Ids is honored. */ @Test (timeout=10000) - public void testSnapshotLimits() throws Exception { - // Setup mock objects for SnapshotManager.createSnapshot. - // + public void testSnapshotIDLimits() throws Exception { + testMaxSnapshotLimit(testMaxSnapshotIDLimit, "rollover", + new Configuration(), testMaxSnapshotIDLimit); + } + + /** + * Tests that the global limit on snapshots is honored. + */ + @Test (timeout=10000) + public void testMaxSnapshotLimit() throws Exception { + Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_FILESYSTEM_LIMIT, + testMaxSnapshotIDLimit); + conf.setInt(DFSConfigKeys. + DFS_NAMENODE_SNAPSHOT_MAX_LIMIT, + testMaxSnapshotIDLimit); + testMaxSnapshotLimit(testMaxSnapshotIDLimit,"max snapshot limit" , + conf, testMaxSnapshotIDLimit * 2); + } + + private void testMaxSnapshotLimit(int maxSnapshotLimit, String errMsg, + Configuration conf, int maxSnapID) + throws IOException { LeaseManager leaseManager = mock(LeaseManager.class); INodeDirectory ids = mock(INodeDirectory.class); FSDirectory fsdir = mock(FSDirectory.class); INodesInPath iip = mock(INodesInPath.class); - SnapshotManager sm = spy(new SnapshotManager(new Configuration(), fsdir)); + SnapshotManager sm = spy(new SnapshotManager(conf, fsdir)); doReturn(ids).when(sm).getSnapshottableRoot(any()); - doReturn(testMaxSnapshotLimit).when(sm).getMaxSnapshotID(); + doReturn(maxSnapID).when(sm).getMaxSnapshotID(); // Create testMaxSnapshotLimit snapshots. These should all succeed. // - for (Integer i = 0; i < testMaxSnapshotLimit; ++i) { + for (Integer i = 0; i < maxSnapshotLimit; ++i) { sm.createSnapshot(leaseManager, iip, "dummy", i.toString(), Time.now()); } @@ -73,7 +96,7 @@ public void testSnapshotLimits() throws Exception { Assert.fail("Expected SnapshotException not thrown"); } catch (SnapshotException se) { Assert.assertTrue( - StringUtils.toLowerCase(se.getMessage()).contains("rollover")); + StringUtils.toLowerCase(se.getMessage()).contains(errMsg)); } // Delete a snapshot to free up a slot. @@ -83,22 +106,26 @@ public void testSnapshotLimits() throws Exception { // Attempt to create a snapshot again. It should still fail due // to snapshot ID rollover. // + try { sm.createSnapshot(leaseManager, iip, "dummy", "shouldFailSnapshot2", Time.now()); - Assert.fail("Expected SnapshotException not thrown"); + // in case the snapshot ID limit is hit, further creation of snapshots + // even post deletions of snapshots won't succeed + if (maxSnapID < maxSnapshotLimit) { + Assert.fail("CreateSnapshot should succeed"); + } } catch (SnapshotException se) { Assert.assertTrue( - StringUtils.toLowerCase(se.getMessage()).contains("rollover")); + StringUtils.toLowerCase(se.getMessage()).contains(errMsg)); } } - /** * Snapshot is identified by INODE CURRENT_STATE_ID. * So maximum allowable snapshotID should be less than CURRENT_STATE_ID */ @Test - public void testValidateSnapshotIDWidth() { + public void testValidateSnapshotIDWidth() throws Exception { FSDirectory fsdir = mock(FSDirectory.class); SnapshotManager snapshotManager = new SnapshotManager(new Configuration(), fsdir); From 8fd4f5490f59a2e9e561b6438b30b3a7453c808b Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Tue, 4 Aug 2020 20:30:02 +0530 Subject: [PATCH 037/335] HADOOP-17131. Refactor S3A Listing code for better isolation. (#2148) Contributed by Mukund Thakur. --- .../org/apache/hadoop/fs/s3a/Listing.java | 191 ++++++++++++++++-- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 158 +++++++-------- .../s3a/impl/ListingOperationCallbacks.java | 119 +++++++++++ .../hadoop/fs/s3a/impl/StoreContext.java | 4 + .../s3a/s3guard/DumpS3GuardDynamoTable.java | 4 +- .../org/apache/hadoop/fs/s3a/TestListing.java | 2 +- .../s3a/impl/TestPartialDeleteFailures.java | 149 +++++++++++++- 7 files changed, 521 insertions(+), 106 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index 9c2f67dd884ec..fcb492857e617 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -30,11 +30,22 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation; +import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks; +import org.apache.hadoop.fs.s3a.impl.StoreContext; +import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata; +import org.apache.hadoop.fs.s3a.s3guard.MetadataStoreListFilesIterator; +import org.apache.hadoop.fs.s3a.s3guard.PathMetadata; +import org.apache.hadoop.fs.s3a.s3guard.S3Guard; import com.google.common.base.Preconditions; import org.slf4j.Logger; +import java.io.FileNotFoundException; import java.io.IOException; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -46,25 +57,31 @@ import java.util.Set; import static org.apache.hadoop.fs.s3a.Constants.S3N_FOLDER_SUFFIX; +import static org.apache.hadoop.fs.s3a.S3AUtils.ACCEPT_ALL; import static org.apache.hadoop.fs.s3a.S3AUtils.createFileStatus; +import static org.apache.hadoop.fs.s3a.S3AUtils.maybeAddTrailingSlash; import static org.apache.hadoop.fs.s3a.S3AUtils.objectRepresentsDirectory; import static org.apache.hadoop.fs.s3a.S3AUtils.stringify; import static org.apache.hadoop.fs.s3a.S3AUtils.translateException; +import static org.apache.hadoop.fs.s3a.auth.RoleModel.pathToKey; /** * Place for the S3A listing classes; keeps all the small classes under control. */ @InterfaceAudience.Private -public class Listing { +public class Listing extends AbstractStoreOperation { - private final S3AFileSystem owner; private static final Logger LOG = S3AFileSystem.LOG; static final FileStatusAcceptor ACCEPT_ALL_BUT_S3N = new AcceptAllButS3nDirs(); - public Listing(S3AFileSystem owner) { - this.owner = owner; + private final ListingOperationCallbacks listingOperationCallbacks; + + public Listing(ListingOperationCallbacks listingOperationCallbacks, + StoreContext storeContext) { + super(storeContext); + this.listingOperationCallbacks = listingOperationCallbacks; } /** @@ -156,6 +173,145 @@ TombstoneReconcilingIterator createTombstoneReconcilingIterator( return new TombstoneReconcilingIterator(iterator, tombstones); } + + /** + * List files under a path assuming the path to be a directory. + * @param path input path. + * @param recursive recursive listing? + * @param acceptor file status filter + * @param collectTombstones should tombstones be collected from S3Guard? + * @param forceNonAuthoritativeMS forces metadata store to act like non + * authoritative. This is useful when + * listFiles output is used by import tool. + * @return an iterator over listing. + * @throws IOException any exception. + */ + public RemoteIterator getListFilesAssumingDir( + Path path, + boolean recursive, Listing.FileStatusAcceptor acceptor, + boolean collectTombstones, + boolean forceNonAuthoritativeMS) throws IOException { + + String key = maybeAddTrailingSlash(pathToKey(path)); + String delimiter = recursive ? null : "/"; + LOG.debug("Requesting all entries under {} with delimiter '{}'", + key, delimiter); + final RemoteIterator cachedFilesIterator; + final Set tombstones; + boolean allowAuthoritative = listingOperationCallbacks + .allowAuthoritative(path); + if (recursive) { + final PathMetadata pm = getStoreContext() + .getMetadataStore() + .get(path, true); + if (pm != null) { + if (pm.isDeleted()) { + OffsetDateTime deletedAt = OffsetDateTime + .ofInstant(Instant.ofEpochMilli( + pm.getFileStatus().getModificationTime()), + ZoneOffset.UTC); + throw new FileNotFoundException("Path " + path + " is recorded as " + + "deleted by S3Guard at " + deletedAt); + } + } + MetadataStoreListFilesIterator metadataStoreListFilesIterator = + new MetadataStoreListFilesIterator( + getStoreContext().getMetadataStore(), + pm, + allowAuthoritative); + tombstones = metadataStoreListFilesIterator.listTombstones(); + // if all of the below is true + // - authoritative access is allowed for this metadatastore + // for this directory, + // - all the directory listings are authoritative on the client + // - the caller does not force non-authoritative access + // return the listing without any further s3 access + if (!forceNonAuthoritativeMS && + allowAuthoritative && + metadataStoreListFilesIterator.isRecursivelyAuthoritative()) { + S3AFileStatus[] statuses = S3Guard.iteratorToStatuses( + metadataStoreListFilesIterator, tombstones); + cachedFilesIterator = createProvidedFileStatusIterator( + statuses, ACCEPT_ALL, acceptor); + return createLocatedFileStatusIterator(cachedFilesIterator); + } + cachedFilesIterator = metadataStoreListFilesIterator; + } else { + DirListingMetadata meta = + S3Guard.listChildrenWithTtl( + getStoreContext().getMetadataStore(), + path, + listingOperationCallbacks.getUpdatedTtlTimeProvider(), + allowAuthoritative); + if (meta != null) { + tombstones = meta.listTombstones(); + } else { + tombstones = null; + } + cachedFilesIterator = createProvidedFileStatusIterator( + S3Guard.dirMetaToStatuses(meta), ACCEPT_ALL, acceptor); + if (allowAuthoritative && meta != null && meta.isAuthoritative()) { + // metadata listing is authoritative, so return it directly + return createLocatedFileStatusIterator(cachedFilesIterator); + } + } + return createTombstoneReconcilingIterator( + createLocatedFileStatusIterator( + createFileStatusListingIterator(path, + listingOperationCallbacks + .createListObjectsRequest(key, delimiter), + ACCEPT_ALL, + acceptor, + cachedFilesIterator)), + collectTombstones ? tombstones : null); + } + + /** + * Generate list located status for a directory. + * Also performing tombstone reconciliation for guarded directories. + * @param dir directory to check. + * @param filter a path filter. + * @return an iterator that traverses statuses of the given dir. + * @throws IOException in case of failure. + */ + public RemoteIterator getLocatedFileStatusIteratorForDir( + Path dir, PathFilter filter) throws IOException { + final String key = maybeAddTrailingSlash(pathToKey(dir)); + final Listing.FileStatusAcceptor acceptor = + new Listing.AcceptAllButSelfAndS3nDirs(dir); + boolean allowAuthoritative = listingOperationCallbacks + .allowAuthoritative(dir); + DirListingMetadata meta = + S3Guard.listChildrenWithTtl(getStoreContext().getMetadataStore(), + dir, + listingOperationCallbacks + .getUpdatedTtlTimeProvider(), + allowAuthoritative); + Set tombstones = meta != null + ? meta.listTombstones() + : null; + final RemoteIterator cachedFileStatusIterator = + createProvidedFileStatusIterator( + S3Guard.dirMetaToStatuses(meta), filter, acceptor); + return (allowAuthoritative && meta != null + && meta.isAuthoritative()) + ? createLocatedFileStatusIterator( + cachedFileStatusIterator) + : createTombstoneReconcilingIterator( + createLocatedFileStatusIterator( + createFileStatusListingIterator(dir, + listingOperationCallbacks + .createListObjectsRequest(key, "/"), + filter, + acceptor, + cachedFileStatusIterator)), + tombstones); + } + + public S3ListRequest createListObjectsRequest(String key, String delimiter) { + return listingOperationCallbacks.createListObjectsRequest(key, delimiter); + } + /** * Interface to implement by the logic deciding whether to accept a summary * entry or path as a valid file or directory. @@ -193,9 +349,9 @@ interface FileStatusAcceptor { * value. * * If the status value is null, the iterator declares that it has no data. - * This iterator is used to handle {@link S3AFileSystem#listStatus} calls - * where the path handed in refers to a file, not a directory: this is the - * iterator returned. + * This iterator is used to handle {@link S3AFileSystem#listStatus(Path)} + * calls where the path handed in refers to a file, not a directory: + * this is the iterator returned. */ static final class SingleStatusRemoteIterator implements RemoteIterator { @@ -465,14 +621,15 @@ private boolean buildNextStatusBatch(S3ListResult objects) { // objects for (S3ObjectSummary summary : objects.getObjectSummaries()) { String key = summary.getKey(); - Path keyPath = owner.keyToQualifiedPath(key); + Path keyPath = getStoreContext().getContextAccessors().keyToPath(key); if (LOG.isDebugEnabled()) { LOG.debug("{}: {}", keyPath, stringify(summary)); } // Skip over keys that are ourselves and old S3N _$folder$ files if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) { S3AFileStatus status = createFileStatus(keyPath, summary, - owner.getDefaultBlockSize(keyPath), owner.getUsername(), + listingOperationCallbacks.getDefaultBlockSize(keyPath), + getStoreContext().getUsername(), summary.getETag(), null); LOG.debug("Adding: {}", status); stats.add(status); @@ -485,10 +642,12 @@ private boolean buildNextStatusBatch(S3ListResult objects) { // prefixes: always directories for (String prefix : objects.getCommonPrefixes()) { - Path keyPath = owner.keyToQualifiedPath(prefix); + Path keyPath = getStoreContext() + .getContextAccessors() + .keyToPath(prefix); if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) { S3AFileStatus status = new S3AFileStatus(Tristate.FALSE, keyPath, - owner.getUsername()); + getStoreContext().getUsername()); LOG.debug("Adding directory: {}", status); added++; stats.add(status); @@ -573,8 +732,8 @@ class ObjectListingIterator implements RemoteIterator { Path listPath, S3ListRequest request) throws IOException { this.listPath = listPath; - this.maxKeys = owner.getMaxKeys(); - this.objects = owner.listObjects(request); + this.maxKeys = listingOperationCallbacks.getMaxKeys(); + this.objects = listingOperationCallbacks.listObjects(request); this.request = request; } @@ -616,7 +775,8 @@ public S3ListResult next() throws IOException { // need to request a new set of objects. LOG.debug("[{}], Requesting next {} objects under {}", listingCount, maxKeys, listPath); - objects = owner.continueListObjects(request, objects); + objects = listingOperationCallbacks + .continueListObjects(request, objects); listingCount++; LOG.debug("New listing status: {}", this); } catch (AmazonClientException e) { @@ -716,7 +876,8 @@ public boolean hasNext() throws IOException { @Override public S3ALocatedFileStatus next() throws IOException { - return owner.toLocatedFileStatus(statusIterator.next()); + return listingOperationCallbacks + .toLocatedFileStatus(statusIterator.next()); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 286df44939312..2cd23255c4b26 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -105,6 +105,7 @@ import org.apache.hadoop.fs.s3a.impl.CopyOutcome; import org.apache.hadoop.fs.s3a.impl.DeleteOperation; import org.apache.hadoop.fs.s3a.impl.InternalConstants; +import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks; import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport; import org.apache.hadoop.fs.s3a.impl.OperationCallbacks; import org.apache.hadoop.fs.s3a.impl.RenameOperation; @@ -148,7 +149,6 @@ import org.apache.hadoop.fs.s3a.select.SelectBinding; import org.apache.hadoop.fs.s3a.select.SelectConstants; import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata; -import org.apache.hadoop.fs.s3a.s3guard.MetadataStoreListFilesIterator; import org.apache.hadoop.fs.s3a.s3guard.MetadataStore; import org.apache.hadoop.fs.s3a.s3guard.PathMetadata; import org.apache.hadoop.fs.s3a.s3guard.S3Guard; @@ -293,6 +293,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private final S3AFileSystem.OperationCallbacksImpl operationCallbacks = new OperationCallbacksImpl(); + private final ListingOperationCallbacks listingOperationCallbacks = + new ListingOperationCallbacksImpl(); + /** Add any deprecated keys. */ @SuppressWarnings("deprecation") private static void addDeprecatedKeys() { @@ -362,7 +365,6 @@ public void initialize(URI name, Configuration originalConf) FAIL_ON_METADATA_WRITE_ERROR_DEFAULT); maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1); - listing = new Listing(this); partSize = getMultipartSizeProperty(conf, MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); multiPartThreshold = getMultipartSizeProperty(conf, @@ -455,6 +457,7 @@ public void initialize(URI name, Configuration originalConf) pageSize = intOption(getConf(), BULK_DELETE_PAGE_SIZE, BULK_DELETE_PAGE_SIZE_DEFAULT, 0); + listing = new Listing(listingOperationCallbacks, createStoreContext()); } catch (AmazonClientException e) { // amazon client exception: stop all services then throw the translation stopAllServices(); @@ -589,6 +592,14 @@ public S3AInstrumentation getInstrumentation() { return instrumentation; } + /** + * Get current listing instance. + * @return this instance's listing. + */ + public Listing getListing() { + return listing; + } + /** * Set up the client bindings. * If delegation tokens are enabled, the FS first looks for a DT @@ -1599,6 +1610,61 @@ public RemoteIterator listObjects( } } + protected class ListingOperationCallbacksImpl implements + ListingOperationCallbacks { + + @Override + @Retries.RetryRaw + public S3ListResult listObjects( + S3ListRequest request) + throws IOException { + return S3AFileSystem.this.listObjects(request); + } + + @Override + @Retries.RetryRaw + public S3ListResult continueListObjects( + S3ListRequest request, + S3ListResult prevResult) + throws IOException { + return S3AFileSystem.this.continueListObjects(request, prevResult); + } + + @Override + public S3ALocatedFileStatus toLocatedFileStatus( + S3AFileStatus status) + throws IOException { + return S3AFileSystem.this.toLocatedFileStatus(status); + } + + @Override + public S3ListRequest createListObjectsRequest( + String key, + String delimiter) { + return S3AFileSystem.this.createListObjectsRequest(key, delimiter); + } + + @Override + public long getDefaultBlockSize(Path path) { + return S3AFileSystem.this.getDefaultBlockSize(path); + } + + @Override + public int getMaxKeys() { + return S3AFileSystem.this.getMaxKeys(); + } + + @Override + public ITtlTimeProvider getUpdatedTtlTimeProvider() { + return S3AFileSystem.this.ttlTimeProvider; + } + + @Override + public boolean allowAuthoritative(final Path p) { + return S3AFileSystem.this.allowAuthoritative(p); + } + } + /** * Low-level call to get at the object metadata. * @param path path to the object @@ -4216,7 +4282,7 @@ private RemoteIterator innerListFiles( // Assuming the path to be a directory // do a bulk operation. RemoteIterator listFilesAssumingDir = - getListFilesAssumingDir(path, + listing.getListFilesAssumingDir(path, recursive, acceptor, collectTombstones, @@ -4242,89 +4308,6 @@ private RemoteIterator innerListFiles( } } - /** - * List files under a path assuming the path to be a directory. - * @param path input path. - * @param recursive recursive listing? - * @param acceptor file status filter - * @param collectTombstones should tombstones be collected from S3Guard? - * @param forceNonAuthoritativeMS forces metadata store to act like non - * authoritative. This is useful when - * listFiles output is used by import tool. - * @return an iterator over listing. - * @throws IOException any exception. - */ - private RemoteIterator getListFilesAssumingDir( - Path path, - boolean recursive, Listing.FileStatusAcceptor acceptor, - boolean collectTombstones, - boolean forceNonAuthoritativeMS) throws IOException { - - String key = maybeAddTrailingSlash(pathToKey(path)); - String delimiter = recursive ? null : "/"; - LOG.debug("Requesting all entries under {} with delimiter '{}'", - key, delimiter); - final RemoteIterator cachedFilesIterator; - final Set tombstones; - boolean allowAuthoritative = allowAuthoritative(path); - if (recursive) { - final PathMetadata pm = metadataStore.get(path, true); - if (pm != null) { - if (pm.isDeleted()) { - OffsetDateTime deletedAt = OffsetDateTime - .ofInstant(Instant.ofEpochMilli( - pm.getFileStatus().getModificationTime()), - ZoneOffset.UTC); - throw new FileNotFoundException("Path " + path + " is recorded as " + - "deleted by S3Guard at " + deletedAt); - } - } - MetadataStoreListFilesIterator metadataStoreListFilesIterator = - new MetadataStoreListFilesIterator(metadataStore, pm, - allowAuthoritative); - tombstones = metadataStoreListFilesIterator.listTombstones(); - // if all of the below is true - // - authoritative access is allowed for this metadatastore - // for this directory, - // - all the directory listings are authoritative on the client - // - the caller does not force non-authoritative access - // return the listing without any further s3 access - if (!forceNonAuthoritativeMS && - allowAuthoritative && - metadataStoreListFilesIterator.isRecursivelyAuthoritative()) { - S3AFileStatus[] statuses = S3Guard.iteratorToStatuses( - metadataStoreListFilesIterator, tombstones); - cachedFilesIterator = listing.createProvidedFileStatusIterator( - statuses, ACCEPT_ALL, acceptor); - return listing.createLocatedFileStatusIterator(cachedFilesIterator); - } - cachedFilesIterator = metadataStoreListFilesIterator; - } else { - DirListingMetadata meta = - S3Guard.listChildrenWithTtl(metadataStore, path, ttlTimeProvider, - allowAuthoritative); - if (meta != null) { - tombstones = meta.listTombstones(); - } else { - tombstones = null; - } - cachedFilesIterator = listing.createProvidedFileStatusIterator( - S3Guard.dirMetaToStatuses(meta), ACCEPT_ALL, acceptor); - if (allowAuthoritative && meta != null && meta.isAuthoritative()) { - // metadata listing is authoritative, so return it directly - return listing.createLocatedFileStatusIterator(cachedFilesIterator); - } - } - return listing.createTombstoneReconcilingIterator( - listing.createLocatedFileStatusIterator( - listing.createFileStatusListingIterator(path, - createListObjectsRequest(key, delimiter), - ACCEPT_ALL, - acceptor, - cachedFilesIterator)), - collectTombstones ? tombstones : null); - } - /** * Override superclass so as to add statistic collection. * {@inheritDoc} @@ -4363,7 +4346,7 @@ public RemoteIterator listLocatedStatus(final Path f, // trigger a list call directly. final RemoteIterator locatedFileStatusIteratorForDir = - getLocatedFileStatusIteratorForDir(path, filter); + listing.getLocatedFileStatusIteratorForDir(path, filter); // If no listing is present then path might be a file. if (!locatedFileStatusIteratorForDir.hasNext()) { @@ -4847,5 +4830,6 @@ public String getBucketLocation() throws IOException { public Path makeQualified(final Path path) { return S3AFileSystem.this.makeQualified(path); } + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java new file mode 100644 index 0000000000000..d89cada84600f --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.Retries; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus; +import org.apache.hadoop.fs.s3a.S3ListRequest; +import org.apache.hadoop.fs.s3a.S3ListResult; +import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider; + +/** + * These are all the callbacks which + * {@link org.apache.hadoop.fs.s3a.Listing} operations + * need, derived from the actual appropriate S3AFileSystem + * methods. + */ +public interface ListingOperationCallbacks { + + /** + * Initiate a {@code listObjects} operation, incrementing metrics + * in the process. + * + * Retry policy: retry untranslated. + * @param request request to initiate + * @return the results + * @throws IOException if the retry invocation raises one (it shouldn't). + */ + @Retries.RetryRaw + S3ListResult listObjects( + S3ListRequest request) + throws IOException; + + /** + * List the next set of objects. + * Retry policy: retry untranslated. + * @param request last list objects request to continue + * @param prevResult last paged result to continue from + * @return the next result object + * @throws IOException none, just there for retryUntranslated. + */ + @Retries.RetryRaw + S3ListResult continueListObjects( + S3ListRequest request, + S3ListResult prevResult) + throws IOException; + + /** + * Build a {@link S3ALocatedFileStatus} from a {@link FileStatus} instance. + * @param status file status + * @return a located status with block locations set up from this FS. + * @throws IOException IO Problems. + */ + S3ALocatedFileStatus toLocatedFileStatus( + S3AFileStatus status) + throws IOException; + /** + * Create a {@code ListObjectsRequest} request against this bucket, + * with the maximum keys returned in a query set by + * {@link this.getMaxKeys()}. + * @param key key for request + * @param delimiter any delimiter + * @return the request + */ + S3ListRequest createListObjectsRequest( + String key, + String delimiter); + + + /** + * Return the number of bytes that large input files should be optimally + * be split into to minimize I/O time. The given path will be used to + * locate the actual filesystem. The full path does not have to exist. + * @param path path of file + * @return the default block size for the path's filesystem + */ + long getDefaultBlockSize(Path path); + + /** + * Get the maximum key count. + * @return a value, valid after initialization + */ + int getMaxKeys(); + + /** + * Get the updated time provider for the current fs instance. + * @return implementation of {@link ITtlTimeProvider} + */ + ITtlTimeProvider getUpdatedTtlTimeProvider(); + + /** + * Is the path for this instance considered authoritative on the client, + * that is: will listing/status operations only be handled by the metastore, + * with no fallback to S3. + * @param p path + * @return true iff the path is authoritative on the client. + */ + boolean allowAuthoritative(Path p); +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java index e307c8db9bf17..cafa22fdec80c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java @@ -210,6 +210,10 @@ public boolean isUseListV1() { return useListV1; } + public ContextAccessors getContextAccessors() { + return contextAccessors; + } + /** * Convert a key to a fully qualified path. * @param key input key diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DumpS3GuardDynamoTable.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DumpS3GuardDynamoTable.java index 536481ac23b7e..b5f29cc7a3163 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DumpS3GuardDynamoTable.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DumpS3GuardDynamoTable.java @@ -348,8 +348,8 @@ protected long dumpRawS3ObjectStore( final CsvFile csv) throws IOException { S3AFileSystem fs = getFilesystem(); Path rootPath = fs.qualify(new Path("/")); - Listing listing = new Listing(fs); - S3ListRequest request = fs.createListObjectsRequest("", null); + Listing listing = fs.getListing(); + S3ListRequest request = listing.createListObjectsRequest("", null); long count = 0; RemoteIterator st = listing.createFileStatusListingIterator(rootPath, request, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java index 1a533bfe64609..34726741835d1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java @@ -68,7 +68,7 @@ public void testTombstoneReconcilingIterator() throws Exception { Path[] allFiles = {parent, liveChild, deletedChild}; Path[] liveFiles = {parent, liveChild}; - Listing listing = new Listing(fs); + Listing listing = fs.getListing(); Collection statuses = new ArrayList<>(); statuses.add(blankFileStatus(parent)); statuses.add(blankFileStatus(liveChild)); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java index c9d872e591f41..7fa03a16cd199 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java @@ -32,8 +32,11 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsResult; import com.amazonaws.services.s3.model.MultiObjectDeleteException; +import com.amazonaws.services.s3.transfer.model.CopyResult; import com.google.common.collect.Lists; import org.assertj.core.api.Assertions; import org.junit.Before; @@ -42,14 +45,21 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3AInputPolicy; import org.apache.hadoop.fs.s3a.S3AInstrumentation; +import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; import org.apache.hadoop.fs.s3a.S3AStorageStatistics; +import org.apache.hadoop.fs.s3a.S3ListRequest; +import org.apache.hadoop.fs.s3a.S3ListResult; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState; import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata; import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider; @@ -230,6 +240,142 @@ private StoreContext createMockStoreContext(boolean multiDelete, .build(); } + private static class MinimalListingOperationCallbacks + implements ListingOperationCallbacks { + @Override + public S3ListResult listObjects(S3ListRequest request) + throws IOException { + return null; + } + + @Override + public S3ListResult continueListObjects( + S3ListRequest request, + S3ListResult prevResult) + throws IOException { + return null; + } + + @Override + public S3ALocatedFileStatus toLocatedFileStatus( + S3AFileStatus status) throws IOException { + return null; + } + + @Override + public S3ListRequest createListObjectsRequest( + String key, + String delimiter) { + return null; + } + + @Override + public long getDefaultBlockSize(Path path) { + return 0; + } + + @Override + public int getMaxKeys() { + return 0; + } + + @Override + public ITtlTimeProvider getUpdatedTtlTimeProvider() { + return null; + } + + @Override + public boolean allowAuthoritative(Path p) { + return false; + } + } + private static class MinimalOperationCallbacks + implements OperationCallbacks { + @Override + public S3ObjectAttributes createObjectAttributes( + Path path, + String eTag, + String versionId, + long len) { + return null; + } + + @Override + public S3ObjectAttributes createObjectAttributes( + S3AFileStatus fileStatus) { + return null; + } + + @Override + public S3AReadOpContext createReadContext( + FileStatus fileStatus) { + return null; + } + + @Override + public void finishRename( + Path sourceRenamed, + Path destCreated) + throws IOException { + + } + + @Override + public void deleteObjectAtPath( + Path path, + String key, + boolean isFile, + BulkOperationState operationState) + throws IOException { + + } + + @Override + public RemoteIterator listFilesAndEmptyDirectories( + Path path, + S3AFileStatus status, + boolean collectTombstones, + boolean includeSelf) + throws IOException { + return null; + } + + @Override + public CopyResult copyFile( + String srcKey, + String destKey, + S3ObjectAttributes srcAttributes, + S3AReadOpContext readContext) + throws IOException { + return null; + } + + @Override + public DeleteObjectsResult removeKeys( + List keysToDelete, + boolean deleteFakeDir, + List undeletedObjectsOnFailure, + BulkOperationState operationState, + boolean quiet) + throws MultiObjectDeleteException, AmazonClientException, + IOException { + return null; + } + + @Override + public boolean allowAuthoritative(Path p) { + return false; + } + + @Override + public RemoteIterator listObjects( + Path path, + String key) + throws IOException { + return null; + } + } + private static class MinimalContextAccessor implements ContextAccessors { @Override @@ -333,7 +479,8 @@ public void delete(final Path path, @Override public void deletePaths(final Collection paths, - @Nullable final BulkOperationState operationState) throws IOException { + @Nullable final BulkOperationState operationState) + throws IOException { deleted.addAll(paths); } From 2986058e7f6fa1b5aab259c64a745b2eedb2febe Mon Sep 17 00:00:00 2001 From: sguggilam Date: Tue, 4 Aug 2020 10:30:06 -0700 Subject: [PATCH 038/335] HADOOP-17164. UGI loginUserFromKeytab doesn't set the last login time (#2178) Contributed by Sandeep Guggilam. Signed-off-by: Mingliang Liu Signed-off-by: Steve Loughran --- .../hadoop/security/UserGroupInformation.java | 9 ++++++ .../security/TestUGILoginFromKeytab.java | 29 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 5269e5a33061a..91b64ade598e5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -529,6 +529,14 @@ private void setLogin(LoginContext login) { user.setLogin(login); } + /** + * Set the last login time for logged in user + * @param loginTime the number of milliseconds since the beginning of time + */ + private void setLastLogin(long loginTime) { + user.setLastLogin(loginTime); + } + /** * Create a UserGroupInformation for the given subject. * This does not change the subject or acquire new credentials. @@ -1968,6 +1976,7 @@ private static UserGroupInformation doSubjectLogin( if (subject == null) { params.put(LoginParam.PRINCIPAL, ugi.getUserName()); ugi.setLogin(login); + ugi.setLastLogin(Time.now()); } return ugi; } catch (LoginException le) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java index 8ede451db964c..d233234c26c0c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java @@ -23,6 +23,7 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -101,12 +102,35 @@ public void stopMiniKdc() { } } + /** + * Login from keytab using the MiniKDC. + */ + @Test + public void testUGILoginFromKeytab() throws Exception { + long beforeLogin = Time.now(); + String principal = "foo"; + File keytab = new File(workDir, "foo.keytab"); + kdc.createPrincipal(keytab, principal); + + UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + Assert.assertTrue("UGI should be configured to login from keytab", + ugi.isFromKeytab()); + + User user = getUser(ugi.getSubject()); + Assert.assertNotNull(user.getLogin()); + + Assert.assertTrue("User login time is less than before login time, " + + "beforeLoginTime:" + beforeLogin + " userLoginTime:" + user.getLastLogin(), + user.getLastLogin() > beforeLogin); + } + /** * Login from keytab using the MiniKDC and verify the UGI can successfully * relogin from keytab as well. This will catch regressions like HADOOP-10786. */ @Test - public void testUGILoginFromKeytab() throws Exception { + public void testUGIReLoginFromKeytab() throws Exception { String principal = "foo"; File keytab = new File(workDir, "foo.keytab"); kdc.createPrincipal(keytab, principal); @@ -122,6 +146,9 @@ public void testUGILoginFromKeytab() throws Exception { final LoginContext login1 = user.getLogin(); Assert.assertNotNull(login1); + // Sleep for 2 secs to have a difference between first and second login + Thread.sleep(2000); + ugi.reloginFromKeytab(); final long secondLogin = user.getLastLogin(); final LoginContext login2 = user.getLogin(); From aa5afa72c517f24f28189577dc611a51e4f3883b Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 4 Aug 2020 23:01:05 +0530 Subject: [PATCH 039/335] YARN-10381. Add application attempt state in AppAttempts RM REST API Contributed by Siddharth Ahuja. Reviewed by Bilwa ST. --- .../resourcemanager/webapp/dao/AppAttemptInfo.java | 7 +++++++ .../webapp/TestRMWebServicesAppAttempts.java | 13 +++++++++---- .../src/site/markdown/ResourceManagerRest.md | 5 ++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index b41dc6b07a886..66381f4e7eeef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -45,6 +46,7 @@ public class AppAttemptInfo { private String nodesBlacklistedBySystem; protected String appAttemptId; private String exportPorts; + private RMAppAttemptState appAttemptState; public AppAttemptInfo() { } @@ -89,6 +91,7 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user, } } this.appAttemptId = attempt.getAppAttemptId().toString(); + this.appAttemptState = attempt.getAppAttemptState(); } } @@ -115,4 +118,8 @@ public String getLogsLink() { public String getAppAttemptId() { return this.appAttemptId; } + + public RMAppAttemptState getAppAttemptState() { + return this.appAttemptState; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java index ee6d0181494f4..df168f757bbb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java @@ -381,7 +381,8 @@ private void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt, WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"), WebServicesTestUtils.getXmlString(element, "nodeId"), WebServicesTestUtils.getXmlString(element, "logsLink"), user, - WebServicesTestUtils.getXmlString(element, "exportPorts")); + WebServicesTestUtils.getXmlString(element, "exportPorts"), + WebServicesTestUtils.getXmlString(element, "appAttemptState")); } } @@ -389,17 +390,19 @@ private void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt, String user) throws Exception { - assertEquals("incorrect number of elements", 11, info.length()); + assertEquals("incorrect number of elements", 12, info.length()); verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"), info.getLong("startTime"), info.getString("containerId"), info.getString("nodeHttpAddress"), info.getString("nodeId"), - info.getString("logsLink"), user, info.getString("exportPorts")); + info.getString("logsLink"), user, info.getString("exportPorts"), + info.getString("appAttemptState")); } private void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id, long startTime, String containerId, String nodeHttpAddress, String - nodeId, String logsLink, String user, String exportPorts) { + nodeId, String logsLink, String user, String exportPorts, + String appAttemptState) { assertEquals("id doesn't match", appAttempt.getAppAttemptId() .getAttemptId(), id); @@ -415,5 +418,7 @@ private void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id, assertTrue( "logsLink doesn't contain user info", logsLink.endsWith("/" + user)); + assertEquals("appAttemptState doesn't match", appAttemptState, appAttempt + .getAppAttemptState().toString()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 879075e794446..375e69ac8cd04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -2265,6 +2265,7 @@ appAttempts: | logsLink | string | The http link to the app attempt logs | | containerId | string | The id of the container for the app attempt | | startTime | long | The start time of the attempt (in ms since epoch) | +| appAttemptState | string | The state of the application attempt - valid values are members of the RMAppAttemptState enum: NEW, SUBMITTED, SCHEDULED, ALLOCATED, LAUNCHED, FAILED, RUNNING, FINISHING, FINISHED, KILLED, ALLOCATED_SAVING, LAUNCHED_UNMANAGED_SAVING, FINAL_SAVING | ### Response Examples @@ -2293,7 +2294,8 @@ Response Body: "startTime" : 1326381444693, "id" : 1, "logsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326821518301_0005_01_000001/user1", - "containerId" : "container_1326821518301_0005_01_000001" + "containerId" : "container_1326821518301_0005_01_000001", + "appAttemptState" : "RUNNING" } ] } @@ -2326,6 +2328,7 @@ Response Body: 1326381444693 container_1326821518301_0005_01_000001 http://host.domain.com:8042/node/containerlogs/container_1326821518301_0005_01_000001/user1 + RUNNING ``` From 0277856738aab9c23f33cd2adcfc743e999887bb Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 4 Aug 2020 23:10:42 +0530 Subject: [PATCH 040/335] YARN-10377. Fix filter index to show apps while clicking on queue in RM UI Contributed by Tarun Parimi. --- .../server/resourcemanager/webapp/CapacitySchedulerPage.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 4a58545f08fbb..47c888d189dd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -651,7 +651,9 @@ public void render(HtmlBlock.Block html) { " q = q.substr(q.lastIndexOf(':') + 2);", " q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';", " }", - " $('#apps').dataTable().fnFilter(q, 4, true);", + // Update this filter column index for queue if new columns are added + // Current index for queue column is 5 + " $('#apps').dataTable().fnFilter(q, 5, true);", " });", " $('#cs').show();", "});").__(). From ed3ab4b87d90450e68f510c158517fc186d2e9e1 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 5 Aug 2020 04:09:14 +0900 Subject: [PATCH 041/335] HADOOP-17179. [JDK 11] Fix javadoc error while detecting Java API link (#2186) Signed-off-by: Mingliang Liu --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 06fcee381d116..373450c822bda 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2401,7 +2401,7 @@ maven-javadoc-plugin ${javadoc.skip.jdk11} - 8 + false -html4 From 49996f67f782dfcf2deaac428a927f0de6c71def Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 4 Aug 2020 18:39:56 -0700 Subject: [PATCH 042/335] HADOOP-17184. Add --mvn-custom-repos parameter to yetus calls (#2193) --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 7b5b15482f226..302dbd04d6c9e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -144,6 +144,7 @@ pipeline { # Dockerfile since we don't want to use the auto-pulled version. YETUS_ARGS+=("--docker") YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + YETUS_ARGS+=("--mvn-custom-repos") # effectively treat dev-suport as a custom maven module YETUS_ARGS+=("--skip-dirs=dev-support") From 58def7cecbed74512798248b89db86aa3e4ca746 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 4 Aug 2020 20:48:45 -0700 Subject: [PATCH 043/335] HDFS-15499. Clean up httpfs/pom.xml to remove aws-java-sdk-s3 exclusion (#2188) Signed-off-by: Akira Ajisaka --- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index b0d7c4b14e1eb..f1d172d1e723d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -118,10 +118,6 @@ org.eclipse.jetty servlet-api-2.5 - - com.amazonaws - aws-java-sdk-s3 - org.eclipse.jdt core @@ -153,10 +149,6 @@ org.eclipse.jetty servlet-api-2.5 - - com.amazonaws - aws-java-sdk-s3 - org.eclipse.jdt core From ac697571a13d3d18293e953ea2648bcd74b9069f Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Wed, 5 Aug 2020 17:10:49 +0530 Subject: [PATCH 044/335] HADOOP-17186. Fixing javadoc in ListingOperationCallbacks (#2196) --- .../apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java index d89cada84600f..5def31bc15c77 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java @@ -77,7 +77,7 @@ S3ALocatedFileStatus toLocatedFileStatus( /** * Create a {@code ListObjectsRequest} request against this bucket, * with the maximum keys returned in a query set by - * {@link this.getMaxKeys()}. + * {@link #getMaxKeys()}. * @param key key for request * @param delimiter any delimiter * @return the request From c566cabd62b57c982a47595c3699649bcdf188ae Mon Sep 17 00:00:00 2001 From: bilaharith <52483117+bilaharith@users.noreply.github.com> Date: Wed, 5 Aug 2020 22:08:13 +0530 Subject: [PATCH 045/335] HADOOP-17163. ABFS: Adding debug log for rename failures - Contributed by Bilahari T H --- .../java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 84d6068648088..513150a6bd43b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -323,6 +323,7 @@ public boolean rename(final Path src, final Path dst) throws IOException { abfsStore.rename(qualifiedSrcPath, qualifiedDstPath); return true; } catch(AzureBlobFileSystemException ex) { + LOG.debug("Rename operation failed. ", ex); checkException( src, ex, From 3f73facd7b32c7e4872aa851c9ad1d3495d0a0fd Mon Sep 17 00:00:00 2001 From: bilaharith <52483117+bilaharith@users.noreply.github.com> Date: Wed, 5 Aug 2020 22:31:04 +0530 Subject: [PATCH 046/335] HADOOP-17149. ABFS: Fixing the testcase ITestGetNameSpaceEnabled - Contributed by Bilahari T H --- .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 23 ------- .../fs/azurebfs/ITestSharedKeyAuth.java | 61 +++++++++++++++++++ 2 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index 4268ff2790b21..29de126c4cc40 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -32,8 +32,6 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; -import org.apache.hadoop.fs.azurebfs.services.AuthType; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -46,7 +44,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -145,26 +142,6 @@ public void testFailedRequestWhenFSNotExist() throws Exception { }); } - @Test - public void testFailedRequestWhenCredentialsNotCorrect() throws Exception { - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); - Configuration config = this.getRawConfiguration(); - config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); - String accountName = this.getAccountName(); - String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName; - // Provide a wrong sharedKey - String secret = config.get(configkKey); - secret = (char) (secret.charAt(0) + 1) + secret.substring(1); - config.set(configkKey, secret); - - AzureBlobFileSystem fs = this.getFileSystem(config); - intercept(AbfsRestOperationException.class, - "\"Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.\", 403", - ()-> { - fs.getIsNamespaceEnabled(); - }); - } - @Test public void testEnsureGetAclCallIsMadeOnceWhenConfigIsInvalid() throws Exception { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java new file mode 100644 index 0000000000000..ab55ffa3fe3c6 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import org.junit.Assume; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +public class ITestSharedKeyAuth extends AbstractAbfsIntegrationTest { + + public ITestSharedKeyAuth() throws Exception { + super(); + } + + @Test + public void testWithWrongSharedKey() throws Exception { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + Configuration config = this.getRawConfiguration(); + config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + false); + String accountName = this.getAccountName(); + String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName; + // a wrong sharedKey + String secret = "XjUjsGherkDpljuyThd7RpljhR6uhsFjhlxRpmhgD12lnj7lhfRn8kgPt5" + + "+MJHS7UJNDER+jn6KP6Jnm2ONQlm=="; + config.set(configkKey, secret); + + AbfsClient abfsClient = this.getFileSystem(config).getAbfsClient(); + intercept(AbfsRestOperationException.class, + "\"Server failed to authenticate the request. Make sure the value of " + + "Authorization header is formed correctly including the " + + "signature.\", 403", + () -> { + abfsClient.getAclStatus("/"); + }); + } + +} From 5edd8b925ef22b83350a21abed6ecc551adb92ee Mon Sep 17 00:00:00 2001 From: Eric Badger Date: Wed, 5 Aug 2020 18:47:49 +0000 Subject: [PATCH 047/335] YARN-4575. ApplicationResourceUsageReport should return ALL reserved resource. Contributed by Bibin Chundatt and Eric Payne. --- .../SchedulerApplicationAttempt.java | 4 +- .../common/fica/FiCaSchedulerApp.java | 3 +- .../TestSchedulerApplicationAttempt.java | 38 +++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 657c03cfd1541..4a8f670b4a6ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -1126,8 +1126,8 @@ public ApplicationResourceUsageReport getResourceUsageReport() { getRunningAggregateAppResourceUsage(); Resource usedResourceClone = Resources.clone( attemptResourceUsage.getAllUsed()); - Resource reservedResourceClone = Resources.clone( - attemptResourceUsage.getReserved()); + Resource reservedResourceClone = + Resources.clone(attemptResourceUsage.getAllReserved()); Resource cluster = rmContext.getScheduler().getClusterResource(); ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index cf6ffd9823525..6542d1ade6655 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -736,7 +736,8 @@ private boolean internalUnreserve(FiCaSchedulerNode node, + " on node " + node + ", currently has " + reservedContainers.size() + " at priority " + schedulerKey.getPriority() - + "; currentReservation " + this.attemptResourceUsage.getReserved() + + "; currentReservation " + + this.attemptResourceUsage.getReserved(node.getPartition()) + " on node-label=" + node.getPartition()); return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index 66698201c2f6d..b1080f7d5fea4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -320,6 +321,43 @@ public void testAppPercentagesOnswitch() throws Exception { 0.0f); } + @Test + public void testAllResourceUsage() throws Exception { + FifoScheduler scheduler = mock(FifoScheduler.class); + when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(0, 0)); + when(scheduler.getResourceCalculator()) + .thenReturn(new DefaultResourceCalculator()); + + ApplicationAttemptId appAttId = createAppAttemptId(0, 0); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getEpoch()).thenReturn(3L); + when(rmContext.getScheduler()).thenReturn(scheduler); + when(rmContext.getYarnConfiguration()).thenReturn(conf); + + final String user = "user1"; + Queue queue = createQueue("test", null); + SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, + user, queue, queue.getAbstractUsersManager(), rmContext); + + // Resource request + Resource requestedResource = Resource.newInstance(1536, 2); + app.attemptResourceUsage.incUsed("X", requestedResource); + app.attemptResourceUsage.incUsed("Y", requestedResource); + Resource r2 = Resource.newInstance(1024, 1); + app.attemptResourceUsage.incReserved("X", r2); + app.attemptResourceUsage.incReserved("Y", r2); + + assertTrue("getUsedResources expected " + Resource.newInstance(3072, 4) + + " but was " + app.getResourceUsageReport().getUsedResources(), + Resources.equals(Resource.newInstance(3072, 4), + app.getResourceUsageReport().getUsedResources())); + assertTrue("getReservedResources expected " + Resource.newInstance(2048, 2) + + " but was " + + app.getResourceUsageReport().getReservedResources(), + Resources.equals(Resource.newInstance(2048, 2), + app.getResourceUsageReport().getReservedResources())); + } + @Test public void testSchedulingOpportunityOverflow() throws Exception { ApplicationAttemptId attemptId = createAppAttemptId(0, 0); From dc5470ae86b96d6f84abd7354c3c056d3627871b Mon Sep 17 00:00:00 2001 From: Wanqiang Ji Date: Thu, 6 Aug 2020 08:35:15 +0800 Subject: [PATCH 048/335] MAPREDUCE-7288. Fix TestLongLong#testRightShift (#2183) --- .../hadoop/examples/pi/math/TestLongLong.java | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java index d6f284e50f768..232c53f4d47cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java @@ -53,32 +53,28 @@ public void testMultiplication() { verifyMultiplication(max, max); } - static void verifyRightShift(long a, long b) { + @Test + public void testRightShift() { + for(int i = 0; i < 1000; i++) { + final long a = nextPositiveLong(); + final long b = nextPositiveLong(); + verifyRightShift(a, b); + } + } + + private static void verifyRightShift(long a, long b) { final LongLong ll = new LongLong().set(a, b); final BigInteger bi = ll.toBigInteger(); - for (int i = 0; i < LongLong.SIZE >> 1; i++) { - final long result = ll.shiftRight(i) & MASK; - final long expected = bi.shiftRight(i).longValue() & MASK; - final String s = String.format( - "\na = %x\nb = %x\nll= " + ll + "\nbi= " + bi.toString(16) + "\n", a, - b); - Assert.assertEquals(s, expected, result); - } - final String s = String.format( "\na = %x\nb = %x\nll= " + ll + "\nbi= " + bi.toString(16) + "\n", a, b); - //System.out.println(s); Assert.assertEquals(s, bi, ll.toBigInteger()); - } - @Test - public void testRightShift() { - for(int i = 0; i < 1000; i++) { - final long a = nextPositiveLong(); - final long b = nextPositiveLong(); - verifyMultiplication(a, b); + for (int i = 0; i < LongLong.SIZE >> 1; i++) { + final long result = ll.shiftRight(i) & MASK; + final long expected = bi.shiftRight(i).longValue() & MASK; + Assert.assertEquals(s, expected, result); } } } From c7e71a6c0beb2748988b339a851a129b5e57f8c4 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 5 Aug 2020 20:51:04 +0530 Subject: [PATCH 049/335] YARN-10361. Make custom DAO classes configurable into RMWebApp#JAXBContextResolver. Contributed by Bilwa ST. --- .../hadoop/yarn/conf/YarnConfiguration.java | 8 ++- .../src/main/resources/yarn-default.xml | 18 +++++- .../webapp/JAXBContextResolver.java | 58 +++++++++++++++++-- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 2d5a59fb2a07f..19ce6221b01fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2398,7 +2398,13 @@ public static boolean isAclEnabled(Configuration conf) { "yarn.http.rmwebapp.external.classes"; public static final String YARN_HTTP_WEBAPP_SCHEDULER_PAGE = - "hadoop.http.rmwebapp.scheduler.page.class"; + "yarn.http.rmwebapp.scheduler.page.class"; + + public static final String YARN_HTTP_WEBAPP_CUSTOM_DAO_CLASSES = + "yarn.http.rmwebapp.custom.dao.classes"; + + public static final String YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES = + "yarn.http.rmwebapp.custom.unwrapped.dao.classes"; /** * Whether or not users are allowed to request that Docker containers honor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 67da860cf56dd..2208b00d8a38e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3355,10 +3355,26 @@ Used to specify custom scheduler page - hadoop.http.rmwebapp.scheduler.page.class + yarn.http.rmwebapp.scheduler.page.class + + + Used to specify custom DAO classes used by custom web services. + + yarn.http.rmwebapp.custom.dao.classes + + + + + + Used to specify custom DAO classes used by custom web services which requires + root unwrapping. + + yarn.http.rmwebapp.custom.unwrapped.dao.classes + + The Node Label script to run. Script output Line starting with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 19aa2015013e6..24428b3bb83de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import com.google.inject.Inject; import com.google.inject.Singleton; import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; @@ -28,6 +29,10 @@ import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*; import org.apache.hadoop.yarn.webapp.RemoteExceptionData; @@ -36,9 +41,17 @@ @Provider public class JAXBContextResolver implements ContextResolver { + private static final Log LOG = + LogFactory.getLog(JAXBContextResolver.class.getName()); + private final Map typesContextMap; public JAXBContextResolver() throws Exception { + this(new Configuration()); + } + + @Inject + public JAXBContextResolver(Configuration conf) throws Exception { JAXBContext context; JAXBContext unWrappedRootContext; @@ -66,17 +79,54 @@ public JAXBContextResolver() throws Exception { DelegationToken.class, AppQueue.class, AppPriority.class, ResourceOptionInfo.class }; + ArrayList finalcTypesList = new ArrayList<>(); + ArrayList finalRootUnwrappedTypesList = new ArrayList<>(); + + Collections.addAll(finalcTypesList, cTypes); + Collections.addAll(finalRootUnwrappedTypesList, rootUnwrappedTypes); + + // Add Custom DAO Classes + Class[] daoClasses = null; + Class[] unwrappedDaoClasses = null; + boolean loadCustom = true; + try { + daoClasses = conf + .getClasses(YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_DAO_CLASSES); + unwrappedDaoClasses = conf.getClasses( + YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES); + } catch (Exception e) { + LOG.warn("Failed to load custom dao class: " + e); + loadCustom = false; + } + + if (loadCustom) { + if (daoClasses != null) { + Collections.addAll(finalcTypesList, daoClasses); + LOG.debug("Added custom dao classes: " + Arrays.toString(daoClasses)); + } + if (unwrappedDaoClasses != null) { + Collections.addAll(finalRootUnwrappedTypesList, unwrappedDaoClasses); + LOG.debug("Added custom Unwrapped dao classes: " + + Arrays.toString(unwrappedDaoClasses)); + } + } + + final Class[] finalcTypes = finalcTypesList + .toArray(new Class[finalcTypesList.size()]); + final Class[] finalRootUnwrappedTypes = finalRootUnwrappedTypesList + .toArray(new Class[finalRootUnwrappedTypesList.size()]); + this.typesContextMap = new HashMap(); context = new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false) - .build(), cTypes); + .build(), finalcTypes); unWrappedRootContext = new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(true) - .build(), rootUnwrappedTypes); - for (Class type : cTypes) { + .build(), finalRootUnwrappedTypes); + for (Class type : finalcTypes) { typesContextMap.put(type, context); } - for (Class type : rootUnwrappedTypes) { + for (Class type : finalRootUnwrappedTypes) { typesContextMap.put(type, unWrappedRootContext); } } From 1d5ccc790bd582f51678e8ab45142107b145ad39 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Fri, 7 Aug 2020 02:17:12 +0900 Subject: [PATCH 050/335] HDFS-15512. Remove smallBufferSize in DFSClient. (#2191) --- .../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index d781dd9ac45db..2fdc716d8d972 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -245,7 +245,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, new DFSHedgedReadMetrics(); private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL; private static volatile ThreadPoolExecutor STRIPED_READ_THREAD_POOL; - private final int smallBufferSize; private final long serverDefaultsValidityPeriod; /** @@ -327,7 +326,6 @@ public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode, this.stats = stats; this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class); this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); - this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); this.dtpReplaceDatanodeOnFailureReplication = (short) conf .getInt(HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure. MIN_REPLICATION, From a2610e21ed5289323d8a6f6359477a8ceb2db2eb Mon Sep 17 00:00:00 2001 From: bilaharith <52483117+bilaharith@users.noreply.github.com> Date: Fri, 7 Aug 2020 03:22:02 +0530 Subject: [PATCH 051/335] HADOOP-17183. ABFS: Enabling checkaccess on ABFS - Contributed by Bilahari T H --- .../constants/FileSystemConfigurations.java | 2 +- .../ITestAzureBlobFileSystemCheckAccess.java | 110 +++++++++++------- 2 files changed, 72 insertions(+), 40 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 260c496434cac..f70d102c1d905 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -84,7 +84,7 @@ public final class FileSystemConfigurations { public static final boolean DEFAULT_ENABLE_HTTPS = true; public static final boolean DEFAULT_USE_UPN = false; - public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = false; + public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = true; public static final boolean DEFAULT_ABFS_LATENCY_TRACK = false; public static final long DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS = 120; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java index 4189d666e7a70..45a3cbd83ccfc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java @@ -17,16 +17,18 @@ */ package org.apache.hadoop.fs.azurebfs; -import com.google.common.collect.Lists; - import java.io.FileNotFoundException; import java.io.IOException; +import java.lang.reflect.Field; import java.util.List; +import com.google.common.collect.Lists; import org.junit.Assume; import org.junit.Test; +import org.mockito.Mockito; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers; @@ -37,6 +39,7 @@ import org.apache.hadoop.security.AccessControlException; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET; @@ -44,9 +47,19 @@ import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_ID; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_SECRET; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Test cases for AzureBlobFileSystem.access() + * + * Some of the tests in this class requires the following 3 configs set in the + * test config file. + * fs.azure.account.test.oauth2.client.id + * fs.azure.account.test.oauth2.client.secret + * fs.azure.check.access.testuser.guid + * Set the above client id, secret and guid of a service principal which has no + * RBAC on the account. + * */ public class ITestAzureBlobFileSystemCheckAccess extends AbstractAbfsIntegrationTest { @@ -66,31 +79,29 @@ public ITestAzureBlobFileSystemCheckAccess() throws Exception { this.isCheckAccessEnabled = getConfiguration().isCheckAccessEnabled(); this.isHNSEnabled = getConfiguration() .getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); + setTestUserFs(); } private void setTestUserFs() throws Exception { if (this.testUserFs != null) { return; } - String orgClientId = getConfiguration().get(FS_AZURE_BLOB_FS_CLIENT_ID); - String orgClientSecret = getConfiguration() - .get(FS_AZURE_BLOB_FS_CLIENT_SECRET); - Boolean orgCreateFileSystemDurungInit = getConfiguration() - .getBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); - getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_ID, - getConfiguration().get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID)); - getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_SECRET, getConfiguration() - .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET)); + final String testClientIdConfKey = + FS_AZURE_BLOB_FS_CLIENT_ID + "." + getAccountName(); + final String testClientId = getConfiguration() + .getString(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID, ""); + getRawConfiguration().set(testClientIdConfKey, testClientId); + final String clientSecretConfKey = + FS_AZURE_BLOB_FS_CLIENT_SECRET + "." + getAccountName(); + final String testClientSecret = getConfiguration() + .getString(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET, ""); + getRawConfiguration().set(clientSecretConfKey, testClientSecret); getRawConfiguration() .setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); - FileSystem fs = FileSystem.newInstance(getRawConfiguration()); - getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_ID, orgClientId); - getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_SECRET, orgClientSecret); - getRawConfiguration() - .setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - orgCreateFileSystemDurungInit); - this.testUserFs = fs; + getRawConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, + AuthType.OAuth.name()); + this.testUserFs = FileSystem.newInstance(getRawConfiguration()); } @Test(expected = IllegalArgumentException.class) @@ -100,15 +111,15 @@ public void testCheckAccessWithNullPath() throws IOException { @Test(expected = NullPointerException.class) public void testCheckAccessForFileWithNullFsAction() throws Exception { - assumeHNSAndCheckAccessEnabled(); + Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false", + isHNSEnabled); // NPE when trying to convert null FsAction enum superUserFs.access(new Path("test.txt"), null); } @Test(expected = FileNotFoundException.class) public void testCheckAccessForNonExistentFile() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path nonExistentFile = setupTestDirectoryAndUserAccess( "/nonExistentFile1.txt", FsAction.ALL); superUserFs.delete(nonExistentFile, true); @@ -153,15 +164,36 @@ public void testCheckAccessForAccountWithoutNS() throws Exception { getConfiguration() .getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true)); Assume.assumeTrue(FS_AZURE_ENABLE_CHECK_ACCESS + " is false", - isCheckAccessEnabled); - setTestUserFs(); + isCheckAccessEnabled); + checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID); + checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET); + checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID); + + // When the driver does not know if the account is HNS enabled or not it + // makes a server call and fails + intercept(AccessControlException.class, + "\"This request is not authorized to perform this operation using " + + "this permission.\", 403", + () -> testUserFs.access(new Path("/"), FsAction.READ)); + + // When the driver has already determined if the account is HNS enabled + // or not, and as the account is non HNS the AzureBlobFileSystem#access + // acts as noop + AzureBlobFileSystemStore mockAbfsStore = + Mockito.mock(AzureBlobFileSystemStore.class); + Mockito.when(mockAbfsStore.getIsNamespaceEnabled()).thenReturn(true); + Field abfsStoreField = AzureBlobFileSystem.class.getDeclaredField( + "abfsStore"); + abfsStoreField.setAccessible(true); + abfsStoreField.set(testUserFs, mockAbfsStore); testUserFs.access(new Path("/"), FsAction.READ); + + superUserFs.access(new Path("/"), FsAction.READ); } @Test public void testFsActionNONE() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path testFilePath = setupTestDirectoryAndUserAccess("/test2.txt", FsAction.NONE); assertInaccessible(testFilePath, FsAction.EXECUTE); @@ -175,8 +207,7 @@ public void testFsActionNONE() throws Exception { @Test public void testFsActionEXECUTE() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path testFilePath = setupTestDirectoryAndUserAccess("/test3.txt", FsAction.EXECUTE); assertAccessible(testFilePath, FsAction.EXECUTE); @@ -191,8 +222,7 @@ public void testFsActionEXECUTE() throws Exception { @Test public void testFsActionREAD() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path testFilePath = setupTestDirectoryAndUserAccess("/test4.txt", FsAction.READ); assertAccessible(testFilePath, FsAction.READ); @@ -207,8 +237,7 @@ public void testFsActionREAD() throws Exception { @Test public void testFsActionWRITE() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path testFilePath = setupTestDirectoryAndUserAccess("/test5.txt", FsAction.WRITE); assertAccessible(testFilePath, FsAction.WRITE); @@ -223,8 +252,7 @@ public void testFsActionWRITE() throws Exception { @Test public void testFsActionREADEXECUTE() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path testFilePath = setupTestDirectoryAndUserAccess("/test6.txt", FsAction.READ_EXECUTE); assertAccessible(testFilePath, FsAction.EXECUTE); @@ -239,8 +267,7 @@ public void testFsActionREADEXECUTE() throws Exception { @Test public void testFsActionWRITEEXECUTE() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path testFilePath = setupTestDirectoryAndUserAccess("/test7.txt", FsAction.WRITE_EXECUTE); assertAccessible(testFilePath, FsAction.EXECUTE); @@ -255,8 +282,7 @@ public void testFsActionWRITEEXECUTE() throws Exception { @Test public void testFsActionALL() throws Exception { - assumeHNSAndCheckAccessEnabled(); - setTestUserFs(); + checkPrerequisites(); Path testFilePath = setupTestDirectoryAndUserAccess("/test8.txt", FsAction.ALL); assertAccessible(testFilePath, FsAction.EXECUTE); @@ -268,13 +294,19 @@ public void testFsActionALL() throws Exception { assertAccessible(testFilePath, FsAction.ALL); } - private void assumeHNSAndCheckAccessEnabled() { + private void checkPrerequisites() { Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false", isHNSEnabled); Assume.assumeTrue(FS_AZURE_ENABLE_CHECK_ACCESS + " is false", isCheckAccessEnabled); + checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID); + checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET); + checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID); + } - Assume.assumeNotNull(getRawConfiguration().get(FS_AZURE_BLOB_FS_CLIENT_ID)); + private void checkIfConfigIsSet(String configKey){ + AbfsConfiguration conf = getConfiguration(); + Assume.assumeNotNull(configKey + " config missing", conf.get(configKey)); } private void assertAccessible(Path testFilePath, FsAction fsAction) From 81da221c757bef9ec35cd190f14b2f872324c661 Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Fri, 7 Aug 2020 08:36:52 +0530 Subject: [PATCH 052/335] YARN-10388. RMNode updatedCapability flag not set while RecommissionNodeTransition. Contributed by Pranjal Protim Borah --- .../yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 3 +++ .../yarn/server/resourcemanager/rmnode/RMNodeImpl.java | 1 + .../yarn/server/resourcemanager/TestRMNodeTransitions.java | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 14b360ca98da7..40f051babd6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -1407,6 +1407,9 @@ public void run() { if (newResource != null) { updateNMResource(newResource); LOG.debug("Node's resource is updated to {}", newResource); + if (!totalResource.equals(newResource)) { + LOG.info("Node's resource is updated to {}", newResource); + } } if (timelineServiceV2Enabled) { updateTimelineCollectorData(response); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 68f44dc6d54e8..55b136411466e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -1234,6 +1234,7 @@ public void transition(RMNodeImpl rmNode, RMNodeEvent event) { if (rmNode.originalTotalCapability != null) { rmNode.totalCapability = rmNode.originalTotalCapability; rmNode.originalTotalCapability = null; + rmNode.updatedCapability = true; } LOG.info("Node " + rmNode.nodeId + " in DECOMMISSIONING is " + "recommissioned back to RUNNING."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index c907cb778ebe6..931a2e77580dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -19,6 +19,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -1062,10 +1064,13 @@ public void testResourceUpdateOnRecommissioningNode() { Resource oldCapacity = node.getTotalCapability(); assertEquals("Memory resource is not match.", oldCapacity.getMemorySize(), 4096); assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4); + assertFalse("updatedCapability should be false.", + node.isUpdatedCapability()); node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.RECOMMISSION)); Resource originalCapacity = node.getOriginalTotalCapability(); assertEquals("Original total capability not null after recommission", null, originalCapacity); + assertTrue("updatedCapability should be set.", node.isUpdatedCapability()); } @Test From 975b6024dd9c7b6e5d0abd3e3f201afd9ae84235 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Fri, 7 Aug 2020 22:19:17 +0530 Subject: [PATCH 053/335] HDFS-15514. Remove useless dfs.webhdfs.enabled. Contributed by Fei Hui. --- .../java/org/apache/hadoop/tools/TestHdfsConfigFields.java | 3 --- .../org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java | 1 - 2 files changed, 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java index 9fd505fd4a706..63c6233d2336b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java @@ -121,9 +121,6 @@ public void initializeMemberVariables() { // Used oddly by DataNode to create new config String xmlPropsToSkipCompare.add("hadoop.hdfs.configuration.version"); - // Skip comparing in branch-2. Removed in trunk with HDFS-7985. - xmlPropsToSkipCompare.add("dfs.webhdfs.enabled"); - // Some properties have moved to HdfsClientConfigKeys xmlPropsToSkipCompare.add("dfs.client.short.circuit.replica.stale.threshold.ms"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java index 7f689e0d3d2d1..6e6ecd1a46144 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java @@ -39,7 +39,6 @@ public MiniDFSClusterService() { @Override protected void serviceInit(Configuration conf) throws Exception { - conf.setBoolean("dfs.webhdfs.enabled", false); super.serviceInit(conf); } From 64753addba9e25a13cc3a932ce9a5d40fd4e998f Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Fri, 7 Aug 2020 17:43:01 -0700 Subject: [PATCH 054/335] YARN-10251. Show extended resources on legacy RM UI. Contributed by Eric Payne --- .../yarn/server/webapp/WebPageUtils.java | 6 +- .../yarn/server/webapp/dao/AppInfo.java | 44 ++++++++--- .../webapp/MetricsOverviewTable.java | 75 +++++++++---------- .../resourcemanager/webapp/NodesPage.java | 22 +++++- .../resourcemanager/webapp/RMAppsBlock.java | 10 +++ .../resourcemanager/webapp/TestNodesPage.java | 4 +- 6 files changed, 104 insertions(+), 57 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index cf4e020d35ec8..311462bd11c2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -61,8 +61,8 @@ private static String getAppsTableColumnDefs( // Update following line if any column added in RM page before column 11 sb.append("{'sType':'num-ignore-str', ") .append("'aTargets': [12, 13, 14, 15, 16] },\n"); - // set progress column index to 19 - progressIndex = "[19]"; + // set progress column index to 21 + progressIndex = "[21]"; } else if (isFairSchedulerPage) { // Update following line if any column added in scheduler page before column 11 sb.append("{'sType':'num-ignore-str', ") @@ -112,4 +112,4 @@ public static String resourceRequestsTableInit() { .toString(); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index d053f33bd0a00..dab3ae2a81297 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -28,9 +28,12 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.StringHelper; @Public @@ -63,8 +66,10 @@ public class AppInfo { protected int priority; private long allocatedCpuVcores; private long allocatedMemoryMB; + private long allocatedGpus; private long reservedCpuVcores; private long reservedMemoryMB; + private long reservedGpus; protected boolean unmanagedApplication; private String appNodeLabelExpression; private String amNodeLabelExpression; @@ -100,24 +105,35 @@ public AppInfo(ApplicationReport app) { if (app.getPriority() != null) { priority = app.getPriority().getPriority(); } - if (app.getApplicationResourceUsageReport() != null) { - runningContainers = app.getApplicationResourceUsageReport() + ApplicationResourceUsageReport usageReport = + app.getApplicationResourceUsageReport(); + if (usageReport != null) { + runningContainers = usageReport .getNumUsedContainers(); - if (app.getApplicationResourceUsageReport().getUsedResources() != null) { - allocatedCpuVcores = app.getApplicationResourceUsageReport() + if (usageReport.getUsedResources() != null) { + allocatedCpuVcores = usageReport .getUsedResources().getVirtualCores(); - allocatedMemoryMB = app.getApplicationResourceUsageReport() + allocatedMemoryMB = usageReport .getUsedResources().getMemorySize(); - reservedCpuVcores = app.getApplicationResourceUsageReport() + reservedCpuVcores = usageReport .getReservedResources().getVirtualCores(); - reservedMemoryMB = app.getApplicationResourceUsageReport() + reservedMemoryMB = usageReport .getReservedResources().getMemorySize(); + Integer gpuIndex = ResourceUtils.getResourceTypeIndex() + .get(ResourceInformation.GPU_URI); + allocatedGpus = -1; + reservedGpus = -1; + if (gpuIndex != null) { + allocatedGpus = usageReport.getUsedResources() + .getResourceValue(ResourceInformation.GPU_URI); + reservedGpus = usageReport.getReservedResources() + .getResourceValue(ResourceInformation.GPU_URI); + } } aggregateResourceAllocation = StringHelper.getResourceSecondsString( - app.getApplicationResourceUsageReport().getResourceSecondsMap()); + usageReport.getResourceSecondsMap()); aggregatePreemptedResourceAllocation = StringHelper - .getResourceSecondsString(app.getApplicationResourceUsageReport() - .getPreemptedResourceSecondsMap()); + .getResourceSecondsString(usageReport.getPreemptedResourceSecondsMap()); } progress = app.getProgress() * 100; // in percent if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { @@ -176,6 +192,10 @@ public long getAllocatedMemoryMB() { return allocatedMemoryMB; } + public long getAllocatedGpus() { + return allocatedGpus; + } + public long getReservedCpuVcores() { return reservedCpuVcores; } @@ -184,6 +204,10 @@ public long getReservedMemoryMB() { return reservedMemoryMB; } + public long getReservedGpus() { + return reservedGpus; + } + public float getProgress() { return progress; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index fbaeafd9218ca..c360c1ae94670 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -19,14 +19,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo; import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; @@ -62,35 +63,34 @@ protected void render(Block html) { DIV div = html.div().$class("metrics"); - long usedMemoryBytes = 0; - long totalMemoryBytes = 0; - long reservedMemoryBytes = 0; - long usedVCores = 0; - long totalVCores = 0; - long reservedVCores = 0; + Resource usedResources; + Resource totalResources; + Resource reservedResources; + int allocatedContainers; if (clusterMetrics.getCrossPartitionMetricsAvailable()) { - ResourceInfo usedAllPartitions = - clusterMetrics.getTotalUsedResourcesAcrossPartition(); - ResourceInfo totalAllPartitions = - clusterMetrics.getTotalClusterResourcesAcrossPartition(); - ResourceInfo reservedAllPartitions = - clusterMetrics.getTotalReservedResourcesAcrossPartition(); - usedMemoryBytes = usedAllPartitions.getMemorySize() * BYTES_IN_MB; - totalMemoryBytes = totalAllPartitions.getMemorySize() * BYTES_IN_MB; - reservedMemoryBytes = reservedAllPartitions.getMemorySize() * BYTES_IN_MB; - usedVCores = usedAllPartitions.getvCores(); - totalVCores = totalAllPartitions.getvCores(); - reservedVCores = reservedAllPartitions.getvCores(); + allocatedContainers = + clusterMetrics.getTotalAllocatedContainersAcrossPartition(); + usedResources = + clusterMetrics.getTotalUsedResourcesAcrossPartition().getResource(); + totalResources = + clusterMetrics.getTotalClusterResourcesAcrossPartition() + .getResource(); + reservedResources = + clusterMetrics.getTotalReservedResourcesAcrossPartition() + .getResource(); // getTotalUsedResourcesAcrossPartition includes reserved resources. - usedMemoryBytes -= reservedMemoryBytes; - usedVCores -= reservedVCores; + Resources.subtractFrom(usedResources, reservedResources); } else { - usedMemoryBytes = clusterMetrics.getAllocatedMB() * BYTES_IN_MB; - totalMemoryBytes = clusterMetrics.getTotalMB() * BYTES_IN_MB; - reservedMemoryBytes = clusterMetrics.getReservedMB() * BYTES_IN_MB; - usedVCores = clusterMetrics.getAllocatedVirtualCores(); - totalVCores = clusterMetrics.getTotalVirtualCores(); - reservedVCores = clusterMetrics.getReservedVirtualCores(); + allocatedContainers = clusterMetrics.getContainersAllocated(); + usedResources = Resource.newInstance( + clusterMetrics.getAllocatedMB() * BYTES_IN_MB, + (int) clusterMetrics.getAllocatedVirtualCores()); + totalResources = Resource.newInstance( + clusterMetrics.getTotalMB() * BYTES_IN_MB, + (int) clusterMetrics.getTotalVirtualCores()); + reservedResources = Resource.newInstance( + clusterMetrics.getReservedMB() * BYTES_IN_MB, + (int) clusterMetrics.getReservedVirtualCores()); } div.h3("Cluster Metrics"). @@ -102,12 +102,9 @@ protected void render(Block html) { th().$class("ui-state-default").__("Apps Running").__(). th().$class("ui-state-default").__("Apps Completed").__(). th().$class("ui-state-default").__("Containers Running").__(). - th().$class("ui-state-default").__("Memory Used").__(). - th().$class("ui-state-default").__("Memory Total").__(). - th().$class("ui-state-default").__("Memory Reserved").__(). - th().$class("ui-state-default").__("VCores Used").__(). - th().$class("ui-state-default").__("VCores Total").__(). - th().$class("ui-state-default").__("VCores Reserved").__(). + th().$class("ui-state-default").__("Used Resources").__(). + th().$class("ui-state-default").__("Total Resources").__(). + th().$class("ui-state-default").__("Reserved Resources").__(). __(). __(). tbody().$class("ui-widget-content"). @@ -121,14 +118,10 @@ protected void render(Block html) { clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled() ) ). - td(String.valueOf( - clusterMetrics.getTotalAllocatedContainersAcrossPartition())). - td(StringUtils.byteDesc(usedMemoryBytes)). - td(StringUtils.byteDesc(totalMemoryBytes)). - td(StringUtils.byteDesc(reservedMemoryBytes)). - td(String.valueOf(usedVCores)). - td(String.valueOf(totalVCores)). - td(String.valueOf(reservedVCores)). + td(String.valueOf(allocatedContainers)). + td(usedResources.toString()). + td(totalResources.toString()). + td(reservedResources.toString()). __(). __().__(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index 8e7974043ed9f..446a81098fb9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -22,6 +22,7 @@ import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; @@ -86,7 +88,9 @@ protected void render(Block html) { .th(".mem", "Mem Used") .th(".mem", "Mem Avail") .th(".vcores", "VCores Used") - .th(".vcores", "VCores Avail"); + .th(".vcores", "VCores Avail") + .th(".gpus", "GPUs Used") + .th(".gpus", "GPUs Avail"); } else { trbody.th(".containers", "Running Containers (G)") .th(".allocationTags", "Allocation Tags") @@ -94,6 +98,8 @@ protected void render(Block html) { .th(".mem", "Mem Avail (G)") .th(".vcores", "VCores Used (G)") .th(".vcores", "VCores Avail (G)") + .th(".gpus", "GPUs Used (G)") + .th(".gpus", "GPUs Avail (G)") .th(".containers", "Running Containers (O)") .th(".mem", "Mem Used (O)") .th(".vcores", "VCores Used (O)") @@ -165,6 +171,16 @@ protected void render(Block html) { nodeTableData.append("\",\"").append(httpAddress).append("\",").append("\""); } + Integer gpuIndex = ResourceUtils.getResourceTypeIndex() + .get(ResourceInformation.GPU_URI); + long usedGPUs = 0; + long availableGPUs = 0; + if (gpuIndex != null) { + usedGPUs = info.getUsedResource().getResource() + .getResourceValue(ResourceInformation.GPU_URI); + availableGPUs = info.getAvailableResource().getResource() + .getResourceValue(ResourceInformation.GPU_URI); + } nodeTableData.append("
") .append(Times.format(info.getLastHealthUpdate())).append("\",\"") @@ -179,6 +195,10 @@ protected void render(Block html) { .append("\",\"").append(String.valueOf(info.getUsedVirtualCores())) .append("\",\"") .append(String.valueOf(info.getAvailableVirtualCores())) + .append("\",\"") + .append(String.valueOf(usedGPUs)) + .append("\",\"") + .append(String.valueOf(availableGPUs)) .append("\",\""); // If opportunistic containers are enabled, add extra fields. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index e8da0cf9ea94e..c90d8ce5dc7dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -69,8 +69,10 @@ public class RMAppsBlock extends AppsBlock { new ColumnHeader(".runningcontainer", "Running Containers"), new ColumnHeader(".allocatedCpu", "Allocated CPU VCores"), new ColumnHeader(".allocatedMemory", "Allocated Memory MB"), + new ColumnHeader(".allocatedGpu", "Allocated GPUs"), new ColumnHeader(".reservedCpu", "Reserved CPU VCores"), new ColumnHeader(".reservedMemory", "Reserved Memory MB"), + new ColumnHeader(".reservedGpu", "Reserved GPUs"), new ColumnHeader(".queuePercentage", "% of Queue"), new ColumnHeader(".clusterPercentage", "% of Cluster"), new ColumnHeader(".progress", "Progress"), @@ -119,6 +121,7 @@ protected void renderData(Block html) { String blacklistedNodesCount = "N/A"; RMApp rmApp = rm.getRMContext().getRMApps() .get(appAttemptId.getApplicationId()); + boolean isAppInCompletedState = false; if (rmApp != null) { RMAppAttempt appAttempt = rmApp.getRMAppAttempt(appAttemptId); Set nodes = @@ -126,6 +129,7 @@ protected void renderData(Block html) { if (nodes != null) { blacklistedNodesCount = String.valueOf(nodes.size()); } + isAppInCompletedState = rmApp.isAppInCompletedStates(); } String percent = StringUtils.format("%.1f", app.getProgress()); appsTableData @@ -171,12 +175,18 @@ protected void renderData(Block html) { .append(app.getAllocatedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getAllocatedMemoryMB())) .append("\",\"") + .append((isAppInCompletedState && app.getAllocatedGpus() <= 0) + ? UNAVAILABLE : String.valueOf(app.getAllocatedGpus())) + .append("\",\"") .append(app.getReservedCpuVcores() == -1 ? "N/A" : String .valueOf(app.getReservedCpuVcores())) .append("\",\"") .append(app.getReservedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getReservedMemoryMB())) .append("\",\"") + .append((isAppInCompletedState && app.getReservedGpus() <= 0) + ? UNAVAILABLE : String.valueOf(app.getReservedGpus())) + .append("\",\"") .append(queuePercent) .append("\",\"") .append(clusterPercent) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 26e8c2ab668d1..1f6a8c0899426 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -48,8 +48,8 @@ public class TestNodesPage { // Number of Actual Table Headers for NodesPage.NodesBlock might change in // future. In that case this value should be adjusted to the new value. - private final int numberOfThInMetricsTable = 23; - private final int numberOfActualTableHeaders = 14; + private final int numberOfThInMetricsTable = 20; + private final int numberOfActualTableHeaders = 16; private final int numberOfThForOpportunisticContainers = 4; private Injector injector; From 40542024df3b9b02b89004dee28c26bae53811f0 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Sat, 8 Aug 2020 15:29:52 +0900 Subject: [PATCH 055/335] HADOOP-17182. Remove breadcrumbs from web site (#2190) Signed-off-by: Mingliang Liu Signed-off-by: Ayush Saxena --- hadoop-project/src/site/site.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 4c9d356e3e95b..86949b0404c50 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -41,11 +41,6 @@ - - - - -

From c2a17659d1d48407edf437131783b653dbc87bc1 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Sat, 8 Aug 2020 14:33:53 +0530 Subject: [PATCH 056/335] HDFS-15443. Setting dfs.datanode.max.transfer.threads to a very small value can cause strange failure. Contributed by AMC-team. --- .../apache/hadoop/hdfs/server/datanode/DataXceiverServer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index 7faf2108bd4fa..43d292c1b551a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -188,6 +188,9 @@ void release() { this.maxXceiverCount = conf.getInt(DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT); + Preconditions.checkArgument(this.maxXceiverCount >= 1, + DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY + + " should not be less than 1."); this.estimateBlockSize = conf.getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); From 5e0f8797790f190c7912082f70e2f715f494ddc4 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 3 Aug 2020 15:09:43 +0530 Subject: [PATCH 057/335] YARN-10364. Fix logic of queue capacity is absolute resource or percentage. Contributed by Bilwa ST. Reviewed by Sunil G. --- .../AbstractAutoCreatedLeafQueue.java | 8 +++ .../scheduler/capacity/AbstractCSQueue.java | 26 +++++----- .../CapacitySchedulerConfiguration.java | 15 ++++++ .../TestAbsoluteResourceConfiguration.java | 51 +++++++++++++++++++ 4 files changed, 88 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index b022b12e274f7..2b22241960618 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -84,6 +84,14 @@ protected Resource getMaximumAbsoluteResource(String queuePath, label); } + @Override + protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, + String label) { + return super.checkConfigTypeIsAbsoluteResource(csContext.getConfiguration() + .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), + label); + } + /** * This methods to change capacity for a queue and adjusts its * absoluteCapacity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index f1467a10626a2..4651611f2add1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -542,6 +542,12 @@ protected Resource getMaximumAbsoluteResource(String queuePath, String label) { return maxResource; } + protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, + String label) { + return csContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label, + queuePath, resourceTypes); + } + protected void updateConfigurableResourceRequirement(String queuePath, Resource clusterResource) { CapacitySchedulerConfiguration conf = csContext.getConfiguration(); @@ -554,17 +560,18 @@ protected void updateConfigurableResourceRequirement(String queuePath, LOG.debug("capacityConfigType is '{}' for queue {}", capacityConfigType, getQueuePath()); + CapacityConfigType localType = checkConfigTypeIsAbsoluteResource( + queuePath, label) ? CapacityConfigType.ABSOLUTE_RESOURCE + : CapacityConfigType.PERCENTAGE; + if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { - this.capacityConfigType = (!minResource.equals(Resources.none()) - && queueCapacities.getAbsoluteCapacity(label) == 0f) - ? CapacityConfigType.ABSOLUTE_RESOURCE - : CapacityConfigType.PERCENTAGE; + this.capacityConfigType = localType; LOG.debug("capacityConfigType is updated as '{}' for queue {}", capacityConfigType, getQueuePath()); + } else { + validateAbsoluteVsPercentageCapacityConfig(localType); } - validateAbsoluteVsPercentageCapacityConfig(minResource); - // If min resource for a resource type is greater than its max resource, // throw exception to handle such error configs. if (!maxResource.equals(Resources.none()) && Resources.greaterThan( @@ -607,12 +614,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, } private void validateAbsoluteVsPercentageCapacityConfig( - Resource minResource) { - CapacityConfigType localType = CapacityConfigType.PERCENTAGE; - if (!minResource.equals(Resources.none())) { - localType = CapacityConfigType.ABSOLUTE_RESOURCE; - } - + CapacityConfigType localType) { if (!queuePath.equals("root") && !this.capacityConfigType.equals(localType)) { throw new IllegalArgumentException("Queue '" + getQueuePath() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 3bebb44a6f64c..96f753315d4e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -2166,6 +2166,21 @@ private void updateMinMaxResourceToConf(String label, String queue, set(prefix, resourceString.toString()); } + public boolean checkConfigTypeIsAbsoluteResource(String label, String queue, + Set resourceTypes) { + String propertyName = getNodeLabelPrefix(queue, label) + CAPACITY; + String resourceString = get(propertyName); + if (resourceString == null || resourceString.isEmpty()) { + return false; + } + + Matcher matcher = RESOURCE_PATTERN.matcher(resourceString); + if (matcher.find()) { + return true; + } + return false; + } + private Resource internalGetLabeledResourceRequirementForQueue(String queue, String label, Set resourceTypes, String suffix) { String propertyName = getNodeLabelPrefix(queue, label) + suffix; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 712d0ada4b857..3d5637c35221a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -524,6 +524,57 @@ public void testComplexValidateAbsoluteResourceConfig() throws Exception { } } + @Test + public void testValidateAbsoluteResourceConfig() throws Exception { + /** + * Queue structure is as follows. root / a / \ a1 a2 + * + * Test below cases: 1) Test ConfigType when resource is [memory=0] + */ + + // create conf with basic queue configuration. + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {QUEUEA, QUEUEB}); + csConf.setQueues(QUEUEA_FULL, new String[] {QUEUEA1, QUEUEA2}); + + // Set default capacities like normal configuration. + csConf.setCapacity(QUEUEA_FULL, "[memory=125]"); + csConf.setCapacity(QUEUEB_FULL, "[memory=0]"); + csConf.setCapacity(QUEUEA1_FULL, "[memory=100]"); + csConf.setCapacity(QUEUEA2_FULL, "[memory=25]"); + + // Update min/max resource to queueA + csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); + csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES); + + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + rm.registerNode("127.0.0.1:1234", 125 * GB, 20); + + // Set [memory=0] to one of the queue and see if reinitialization + // doesnt throw exception saying "Parent queue 'root.A' and + // child queue 'root.A.A2' should use either percentage + // based capacityconfiguration or absolute resource together for label" + csConf.setCapacity(QUEUEA1_FULL, "[memory=125]"); + csConf.setCapacity(QUEUEA2_FULL, "[memory=0]"); + + // Get queue object to verify min/max resource configuration. + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + try { + cs.reinitialize(csConf, rm.getRMContext()); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } + @Test public void testEffectiveResourceAfterReducingClusterResource() throws Exception { From 9062814bab7d6087c584e03b0b16eaa60b31f60a Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 3 Aug 2020 14:39:13 +0900 Subject: [PATCH 058/335] HDFS-15508. [JDK 11] Fix javadoc errors in hadoop-hdfs-rbf module --- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 1 - .../server/federation/router/security/token/package-info.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 489458ca97352..782f28a0ef30e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -31,7 +31,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdfs - true diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java index a51e4552955a5..ae65c8fe6755d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/package-info.java @@ -18,7 +18,8 @@ /** * Includes implementations of token secret managers. - * Implementations should extend {@link AbstractDelegationTokenSecretManager}. + * Implementations should extend + * {@link org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager}. */ @InterfaceAudience.Private @InterfaceStability.Evolving From 7938ebfb9dd12b874ced1798f0c3ad0c160cb465 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 11 Aug 2020 11:14:58 +0900 Subject: [PATCH 059/335] HDFS-15506. [JDK 11] Fix javadoc errors in hadoop-hdfs module. Contributed by Xieming Li. --- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 - .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../src/main/java/org/apache/hadoop/hdfs/DFSUtil.java | 2 -- .../server/blockmanagement/DatanodeAdminDefaultMonitor.java | 2 +- .../namenode/snapshot/DirectorySnapshottableFeature.java | 3 ++- .../src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 1 - 6 files changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 5b50062a31666..e1ea5840deda5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -32,7 +32,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdfs true - true diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index d1c32f122eca5..07ab0aa3269ab 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -685,7 +685,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { 600; /** * The maximum number of getBlocks RPCs data movement utilities can make to - * a NameNode per second. Values <= 0 disable throttling. This affects + * a NameNode per second. Values <= 0 disable throttling. This affects * anything that uses a NameNodeConnector, i.e., the Balancer, Mover, * and StoragePolicySatisfier. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 00f14cd82e42f..f62c1bf45edc9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -39,7 +39,6 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.net.InetAddress; @@ -1777,7 +1776,6 @@ public static DelegationTokenIdentifier decodeDelegationToken( * was found. * @throws ParentNotDirectoryException * @throws UnresolvedLinkException - * @throws FileNotFoundException */ public static void checkProtectedDescendants( FSDirectory fsd, INodesInPath iip) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index a5650d1c4865b..508dab660bd60 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -40,7 +40,7 @@ /** * Checks to see if datanodes have finished DECOMMISSION_INPROGRESS or * ENTERING_MAINTENANCE state. - *

+ *

* Since this is done while holding the namesystem lock, * the amount of work per monitor tick is limited. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index 18ef9a1c68134..3bf2df470429b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -173,7 +173,8 @@ void addSnapshot(Snapshot snapshot) { * Add a snapshot. * @param snapshotRoot Root of the snapshot. * @param name Name of the snapshot. - * @param mtime The snapshot creation time set by Time.now(). + * @param leaseManager + * @param captureOpenFiles * @throws SnapshotException Throw SnapshotException when there is a snapshot * with the same name already exists or snapshot quota exceeds */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index ec5fa0afddc60..a0d30125fcfb0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -2184,7 +2184,6 @@ private static void printUsage(String cmd) { /** * @param argv The parameters passed to this program. - * @exception Exception if the filesystem does not exist. * @return 0 on success, non zero on error. */ @Override From 32895f4f7ea51522e779e68a030bfc9515f40798 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 11 Aug 2020 13:49:56 +0900 Subject: [PATCH 060/335] HDFS-15507. [JDK 11] Fix javadoc errors in hadoop-hdfs-client module. Contributed by Xieming Li. --- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 - .../src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java | 2 +- .../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 2 +- .../java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java | 3 ++- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index c4bc07f29c431..d65e6030369b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -31,7 +31,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hdfs - true diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java index 9b324bd1b07ab..4de969642d574 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java @@ -29,7 +29,7 @@ /** * Global State Id context for the client. - *

+ *

* This is the client side implementation responsible for receiving * state alignment info from server(s). */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 2fdc716d8d972..e9b5dd2b0d208 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1241,7 +1241,7 @@ public DFSOutputStream create(String src, FsPermission permission, /** * Same as {@link #create(String, FsPermission, EnumSet, boolean, short, long, - * addition of Progressable, int, ChecksumOpt, InetSocketAddress[], String)} + * Progressable, int, ChecksumOpt, InetSocketAddress[], String)} * with the storagePolicy that is used to specify a specific storage policy * instead of inheriting any policy from this new file's parent directory. * This policy will be persisted in HDFS. A value of null means inheriting diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index a025b9bad2e69..3a5cdfb57a5eb 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.util.StringUtils; @InterfaceAudience.Private @@ -158,7 +159,7 @@ public byte value() { * period, no other client can write to the file. The writing client can * periodically renew the lease. When the file is closed, the lease is * revoked. The lease duration is bound by this soft limit and a - * {@link HdfsConstants#LEASE_HARDLIMIT_PERIOD hard limit}. Until the + * {@link HdfsClientConfigKeys#DFS_LEASE_HARDLIMIT_KEY }. Until the * soft limit expires, the writer has sole write access to the file. If the * soft limit expires and the client fails to close the file or renew the * lease, another client can preempt the lease. From 592127bdc20cb1232dbb91b9742468281f78a03f Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Tue, 11 Aug 2020 00:00:30 -0700 Subject: [PATCH 061/335] HDFS-15520 Use visitor pattern to visit namespace tree (#2203) --- .../hadoop/hdfs/server/namenode/INode.java | 11 +- .../hdfs/server/namenode/INodeDirectory.java | 6 + .../hdfs/server/namenode/INodeFile.java | 6 + .../hdfs/server/namenode/INodeReference.java | 10 +- .../hdfs/server/namenode/INodeSymlink.java | 9 +- .../namenode/INodeWithAdditionalFields.java | 2 +- .../DirectoryWithSnapshotFeature.java | 2 +- .../visitor/NamespacePrintVisitor.java | 227 ++++++++++++++++ .../namenode/visitor/NamespaceVisitor.java | 243 ++++++++++++++++++ .../hdfs/server/namenode/visitor/package.html | 21 ++ .../namenode/TestFSImageWithSnapshot.java | 4 + 11 files changed, 535 insertions(+), 6 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespaceVisitor.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/package.html diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index fd416f72a3a20..0e8c3a4a9935d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.ChunkedArrayList; @@ -76,7 +77,7 @@ final boolean isRoot() { } /** Get the {@link PermissionStatus} */ - abstract PermissionStatus getPermissionStatus(int snapshotId); + public abstract PermissionStatus getPermissionStatus(int snapshotId); /** The same as getPermissionStatus(null). */ final PermissionStatus getPermissionStatus() { @@ -1123,6 +1124,14 @@ public void clear() { } } + /** Accept a visitor to visit this {@link INode}. */ + public void accept(NamespaceVisitor visitor, int snapshot) { + final Class clazz = visitor != null? visitor.getClass() + : NamespaceVisitor.class; + throw new UnsupportedOperationException(getClass().getSimpleName() + + " does not support " + clazz.getSimpleName()); + } + /** * INode feature such as {@link FileUnderConstructionFeature} * and {@link DirectoryWithQuotaFeature}. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 5f44da8792b8b..3836d79c6a8c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList; @@ -996,6 +997,11 @@ public SnapshotAndINode(int snapshot, INode inode) { } } + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitDirectoryRecursively(this, snapshot); + } + public final int getChildrenNum(final int snapshotId) { return getChildrenList(snapshotId).size(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 3f0208c4dfb0c..eb4042957f3a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.DiffList; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.hdfs.util.LongBitFormat; import org.apache.hadoop.util.StringUtils; import static org.apache.hadoop.io.erasurecode.ErasureCodeConstants.REPLICATION_POLICY_ID; @@ -1111,6 +1112,11 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, out.println(); } + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitFile(this, snapshot); + } + /** * Remove full blocks at the end file up to newLength * @return sum of sizes of the remained blocks diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index 69a92706ab50b..a9941677134ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.security.AccessControlException; /** @@ -368,7 +369,12 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, b.append("->"); getReferredINode().dumpTreeRecursively(out, b, snapshot); } - + + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitReferenceRecursively(this, snapshot); + } + public int getDstSnapshotId() { return Snapshot.CURRENT_STATE_ID; } @@ -399,7 +405,7 @@ public WithCount(INodeReference parent, INode referred) { INodeReferenceValidation.add(this, WithCount.class); } - private String getCountDetails() { + public String getCountDetails() { final StringBuilder b = new StringBuilder("["); if (!withNameList.isEmpty()) { final Iterator i = withNameList.iterator(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java index c76bea090f165..45f4d86edd4e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; /** * An {@link INode} representing a symbolic link. @@ -104,7 +105,13 @@ public ContentSummaryComputationContext computeContentSummary(int snapshotId, public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, final int snapshot) { super.dumpTreeRecursively(out, prefix, snapshot); - out.println(); + out.print(" ~> "); + out.println(getSymlinkString()); + } + + @Override + public void accept(NamespaceVisitor visitor, int snapshot) { + visitor.visitSymlink(this, snapshot); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java index b7d2f2c1e5abc..e0c6c6422ff25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java @@ -178,7 +178,7 @@ final void clonePermissionStatus(INodeWithAdditionalFields that) { } @Override - final PermissionStatus getPermissionStatus(int snapshotId) { + public final PermissionStatus getPermissionStatus(int snapshotId) { return new PermissionStatus(getUserName(snapshotId), getGroupName(snapshotId), getFsPermission(snapshotId)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index f76738ffa2924..b8f7b65ea7e05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -166,7 +166,7 @@ void setSnapshotRoot(INodeDirectoryAttributes root) { this.isSnapshotRoot = true; } - boolean isSnapshotRoot() { + public boolean isSnapshotRoot() { return isSnapshotRoot; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java new file mode 100644 index 0000000000000..0294e62bbdf38 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.visitor; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.INodeSymlink; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; +import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * To print the namespace tree recursively for testing. + * + * \- foo (INodeDirectory@33dd2717) + * \- sub1 (INodeDirectory@442172) + * +- file1 (INodeFile@78392d4) + * +- file2 (INodeFile@78392d5) + * +- sub11 (INodeDirectory@8400cff) + * \- file3 (INodeFile@78392d6) + * \- z_file4 (INodeFile@45848712) + */ +public class NamespacePrintVisitor implements NamespaceVisitor { + static final String NON_LAST_ITEM = "+-"; + static final String LAST_ITEM = "\\-"; + + /** Print the tree from the given root to a {@link File}. */ + public static void print2File(INode root, File f) throws IOException { + try(final PrintWriter out = new PrintWriter(new FileWriter(f), true)) { + new NamespacePrintVisitor(out).print(root); + } + } + + /** @return string of the tree in the given {@link FSNamesystem}. */ + public static String print2Sting(FSNamesystem ns) { + return print2Sting(ns.getFSDirectory().getRoot()); + } + + /** @return string of the tree from the given root. */ + public static String print2Sting(INode root) { + final StringWriter out = new StringWriter(); + new NamespacePrintVisitor(new PrintWriter(out)).print(root); + return out.getBuffer().toString(); + } + + /** + * Print the tree in the given {@link FSNamesystem} + * to the given {@link PrintStream}. + */ + public static void print(FSNamesystem ns, PrintStream out) { + new NamespacePrintVisitor(new PrintWriter(out)).print(ns); + } + + private final PrintWriter out; + private final StringBuffer prefix = new StringBuffer(); + + private NamespacePrintVisitor(PrintWriter out) { + this.out = out; + } + + private void print(FSNamesystem namesystem) { + print(namesystem.getFSDirectory().getRoot()); + } + + private void print(INode root) { + root.accept(this, Snapshot.CURRENT_STATE_ID); + } + + private void printINode(INode iNode, int snapshot) { + out.print(prefix); + out.print(" "); + final String name = iNode.getLocalName(); + out.print(name != null && name.isEmpty()? "/": name); + out.print(" ("); + out.print(iNode.getObjectString()); + out.print("), "); + out.print(iNode.getParentString()); + out.print(", " + iNode.getPermissionStatus(snapshot)); + } + + @Override + public void visitFile(INodeFile file, int snapshot) { + printINode(file, snapshot); + + out.print(", fileSize=" + file.computeFileSize(snapshot)); + // print only the first block, if it exists + out.print(", blocks="); + final BlockInfo[] blocks = file.getBlocks(); + out.print(blocks.length == 0 ? null: blocks[0]); + out.println(); + + final FileWithSnapshotFeature snapshotFeature + = file.getFileWithSnapshotFeature(); + if (snapshotFeature != null) { + if (prefix.length() >= 2) { + prefix.setLength(prefix.length() - 2); + prefix.append(" "); + } + out.print(prefix); + out.print(snapshotFeature); + } + out.println(); + } + + @Override + public void visitSymlink(INodeSymlink symlink, int snapshot) { + printINode(symlink, snapshot); + out.print(" ~> "); + out.println(symlink.getSymlinkString()); + } + + @Override + public void visitReference(INodeReference ref, int snapshot) { + printINode(ref, snapshot); + + if (ref instanceof INodeReference.DstReference) { + out.print(", dstSnapshotId=" + ref.getDstSnapshotId()); + } else if (ref instanceof INodeReference.WithCount) { + out.print(", " + ((INodeReference.WithCount)ref).getCountDetails()); + } + out.println(); + } + + @Override + public void preVisitReferred(INode referred) { + prefix.setLength(prefix.length() - 2); + prefix.append(" ->"); + } + + @Override + public void postVisitReferred(INode referred) { + prefix.setLength(prefix.length() - 2); + } + + @Override + public void visitDirectory(INodeDirectory dir, int snapshot) { + printINode(dir, snapshot); + + out.print(", childrenSize=" + dir.getChildrenList(snapshot).size()); + final DirectoryWithQuotaFeature q = dir.getDirectoryWithQuotaFeature(); + if (q != null) { + out.print(", " + q); + } + if (dir instanceof Snapshot.Root) { + out.print(", snapshotId=" + snapshot); + } + out.println(); + + if (prefix.length() >= 2) { + prefix.setLength(prefix.length() - 2); + prefix.append(" "); + } + + final DirectoryWithSnapshotFeature snapshotFeature + = dir.getDirectoryWithSnapshotFeature(); + if (snapshotFeature != null) { + out.print(prefix); + out.print(snapshotFeature); + } + out.println(); + } + + @Override + public void visitSnapshottable(INodeDirectory dir, + DirectorySnapshottableFeature snapshottable) { + out.println(); + out.print(prefix); + + out.print("Snapshot of "); + final String name = dir.getLocalName(); + out.print(name != null && name.isEmpty()? "/": name); + out.print(": quota="); + out.print(snapshottable.getSnapshotQuota()); + + int n = 0; + for(DirectoryDiff diff : snapshottable.getDiffs()) { + if (diff.isSnapshotRoot()) { + n++; + } + } + final int numSnapshots = snapshottable.getNumSnapshots(); + Preconditions.checkState(n == numSnapshots, + "numSnapshots = " + numSnapshots + " != " + n); + out.print(", #snapshot="); + out.println(n); + } + + @Override + public void preVisitSub(Element sub, int index, boolean isLast) { + prefix.append(isLast? LAST_ITEM : NON_LAST_ITEM); + } + + @Override + public void postVisitSub(Element sub, int index, boolean isLast) { + prefix.setLength(prefix.length() - 2); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespaceVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespaceVisitor.java new file mode 100644 index 0000000000000..2360b59373b22 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespaceVisitor.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.visitor; + +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.INodeSymlink; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; + +import java.util.Iterator; + +/** + * For visiting namespace trees. + */ +public interface NamespaceVisitor { + /** For visiting any {@link INode}. */ + interface INodeVisitor { + INodeVisitor DEFAULT = new INodeVisitor() {}; + + /** Visiting the given {@link INode}. */ + default void visit(INode iNode, int snapshot) { + } + } + + /** @return the default (non-recursive) {@link INodeVisitor}. */ + default INodeVisitor getDefaultVisitor() { + return INodeVisitor.DEFAULT; + } + + /** Visiting the given {@link INodeFile}. */ + default void visitFile(INodeFile file, int snapshot) { + getDefaultVisitor().visit(file, snapshot); + } + + /** Visiting the given {@link INodeSymlink}. */ + default void visitSymlink(INodeSymlink symlink, int snapshot) { + getDefaultVisitor().visit(symlink, snapshot); + } + + /** Visiting the given {@link INodeReference} (non-recursively). */ + default void visitReference(INodeReference ref, int snapshot) { + getDefaultVisitor().visit(ref, snapshot); + } + + /** First visit the given {@link INodeReference} and then the referred. */ + default void visitReferenceRecursively(INodeReference ref, int snapshot) { + visitReference(ref, snapshot); + + final INode referred = ref.getReferredINode(); + preVisitReferred(referred); + referred.accept(this, snapshot); + postVisitReferred(referred); + } + + /** Right before visiting the given referred {@link INode}. */ + default void preVisitReferred(INode referred) { + } + + /** Right after visiting the given referred {@link INode}. */ + default void postVisitReferred(INode referred) { + } + + /** Visiting the given {@link INodeDirectory} (non-recursively). */ + default void visitDirectory(INodeDirectory dir, int snapshot) { + getDefaultVisitor().visit(dir, snapshot); + } + + /** + * First visit the given {@link INodeDirectory}; + * then the children; + * and then, if snapshottable, the snapshots. */ + default void visitDirectoryRecursively(INodeDirectory dir, int snapshot) { + visitDirectory(dir, snapshot); + visitSubs(getChildren(dir, snapshot)); + + if (snapshot == Snapshot.CURRENT_STATE_ID) { + final DirectorySnapshottableFeature snapshottable + = dir.getDirectorySnapshottableFeature(); + if (snapshottable != null) { + visitSnapshottable(dir, snapshottable); + visitSubs(getSnapshots(snapshottable)); + } + } + } + + /** + * Right before visiting the given sub {@link Element}. + * The sub element may be a child of an {@link INodeDirectory} + * or a snapshot in {@link DirectorySnapshottableFeature}. + * + * @param sub the element to be visited. + * @param index the index of the sub element. + * @param isLast is the sub element the last element? + */ + default void preVisitSub(Element sub, int index, boolean isLast) { + } + + /** + * Right after visiting the given sub {@link Element}. + * The sub element may be a child of an {@link INodeDirectory} + * or a snapshot in {@link DirectorySnapshottableFeature}. + * + * @param sub the element just visited. + * @param index the index of the sub element. + * @param isLast is the sub element the last element? + */ + default void postVisitSub(Element sub, int index, boolean isLast) { + } + + /** Visiting a {@link DirectorySnapshottableFeature}. */ + default void visitSnapshottable(INodeDirectory dir, + DirectorySnapshottableFeature snapshottable) { + } + + /** + * Visiting the sub {@link Element}s recursively. + * + * @param subs the children of an {@link INodeDirectory} + * or the snapshots in {@link DirectorySnapshottableFeature}. + */ + default void visitSubs(Iterable subs) { + if (subs == null) { + return; + } + int index = 0; + for(final Iterator i = subs.iterator(); i.hasNext();) { + final Element e = i.next(); + final boolean isList = !i.hasNext(); + preVisitSub(e, index, isList); + e.getInode().accept(this, e.getSnapshotId()); + postVisitSub(e, index, isList); + index++; + } + } + + /** @return the children as {@link Element}s. */ + static Iterable getChildren(INodeDirectory dir, int snapshot) { + final Iterator i = dir.getChildrenList(snapshot).iterator(); + return new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + @Override + public boolean hasNext() { + return i.hasNext(); + } + + @Override + public Element next() { + return new Element(snapshot, i.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + + /** @return the snapshots as {@link Element}s. */ + static Iterable getSnapshots( + DirectorySnapshottableFeature snapshottable) { + final Iterator i + = snapshottable.getDiffs().iterator(); + return new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + private DirectoryWithSnapshotFeature.DirectoryDiff next = findNext(); + + private DirectoryWithSnapshotFeature.DirectoryDiff findNext() { + for(; i.hasNext();) { + final DirectoryWithSnapshotFeature.DirectoryDiff diff = i.next(); + if (diff.isSnapshotRoot()) { + return diff; + } + } + return null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Element next() { + final int id = next.getSnapshotId(); + final Element e = new Element(id, + snapshottable.getSnapshotById(id).getRoot()); + next = findNext(); + return e; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + + /** Snapshot and INode. */ + class Element { + private final int snapshotId; + private final INode inode; + + Element(int snapshot, INode inode) { + this.snapshotId = snapshot; + this.inode = inode; + } + + INode getInode() { + return inode; + } + + int getSnapshotId() { + return snapshotId; + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/package.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/package.html new file mode 100644 index 0000000000000..f25748136d6f4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/package.html @@ -0,0 +1,21 @@ + + + +

Use the visitor pattern to visit namespace tree.

+ + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java index 3fd725b7f6965..de527f0bff763 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; +import org.apache.hadoop.hdfs.server.namenode.visitor.NamespacePrintVisitor; import org.apache.hadoop.hdfs.util.Canceler; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; @@ -607,6 +608,9 @@ String printTree(String label) throws Exception { final String b = fsn.getFSDirectory().getINode("/").dumpTreeRecursively().toString(); output.println(b); + + final String s = NamespacePrintVisitor.print2Sting(fsn); + Assert.assertEquals(b, s); return b; } From 8955a6ceb738b3453a6cfa2fabb5790fb28b1d71 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Tue, 11 Aug 2020 00:01:58 -0700 Subject: [PATCH 062/335] HDFS-15515: mkdirs on fallback should throw IOE out instead of suppressing and returning false (#2205) * HDFS-15515: mkdirs on fallback should throw IOE out instead of suppressing and returning false * Used LambdaTestUtils#intercept in test --- .../java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 7 +++---- .../hadoop/fs/viewfs/TestViewFileSystemLinkFallback.java | 5 ++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index baf0027fa6a84..ad62f94ec6297 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1421,7 +1421,7 @@ public FsStatus getStatus(Path p) throws IOException { @Override public boolean mkdirs(Path dir, FsPermission permission) - throws AccessControlException, FileAlreadyExistsException { + throws IOException { if (theInternalDir.isRoot() && dir == null) { throw new FileAlreadyExistsException("/ already exits"); } @@ -1451,7 +1451,7 @@ public boolean mkdirs(Path dir, FsPermission permission) .append(linkedFallbackFs.getUri()); LOG.debug(msg.toString(), e); } - return false; + throw e; } } @@ -1459,8 +1459,7 @@ public boolean mkdirs(Path dir, FsPermission permission) } @Override - public boolean mkdirs(Path dir) - throws AccessControlException, FileAlreadyExistsException { + public boolean mkdirs(Path dir) throws IOException { return mkdirs(dir, null); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkFallback.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkFallback.java index bd2b5af02ad87..e7317608147be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkFallback.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkFallback.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -759,7 +760,9 @@ public void testMkdirsShouldReturnFalseWhenFallbackFSNotAvailable() cluster.shutdownNameNodes(); // Stopping fallback server // /user1/test1 does not exist in mount internal dir tree, it would // attempt to create in fallback. - assertFalse(vfs.mkdirs(nextLevelToInternalDir)); + intercept(IOException.class, () -> { + vfs.mkdirs(nextLevelToInternalDir); + }); cluster.restartNameNodes(); // should return true succeed when fallback fs is back to normal. assertTrue(vfs.mkdirs(nextLevelToInternalDir)); From 909f1e82d3e956bdd01b9d24ed2108f93bb94953 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Tue, 11 Aug 2020 13:05:08 +0530 Subject: [PATCH 063/335] HADOOP-17196. Fix C/C++ standard warnings (#2208) * Passing C/C++ standard flags -std is not cross-compiler friendly as not all compilers support all values. * Thus, we need to make use of the appropriate flags provided by CMake in order to specify the C/C++ standards. Signed-off-by: Akira Ajisaka --- hadoop-common-project/hadoop-common/HadoopCommon.cmake | 4 ++-- .../hadoop-hdfs-native-client/src/CMakeLists.txt | 3 ++- .../src/main/native/libhdfspp/CMakeLists.txt | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/HadoopCommon.cmake b/hadoop-common-project/hadoop-common/HadoopCommon.cmake index 4de70ac3f78e2..7628ecf628de6 100644 --- a/hadoop-common-project/hadoop-common/HadoopCommon.cmake +++ b/hadoop-common-project/hadoop-common/HadoopCommon.cmake @@ -193,7 +193,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS") # Solaris flags. 64-bit compilation is mandatory, and is checked earlier. hadoop_add_compiler_flags("-m64 -D_POSIX_C_SOURCE=200112L -D__EXTENSIONS__ -D_POSIX_PTHREAD_SEMANTICS") - set(CMAKE_CXX_FLAGS "-std=gnu++98 ${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_STANDARD 98) hadoop_add_linker_flags("-m64") # CMAKE_SYSTEM_PROCESSOR is set to the output of 'uname -p', which on Solaris is @@ -212,4 +212,4 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS") endif() # Set GNU99 as the C standard to use -set(CMAKE_C_FLAGS "-std=gnu99 ${CMAKE_C_FLAGS}") \ No newline at end of file +set(CMAKE_C_STANDARD 99) \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt index 6e233fd3991d6..6b8a795204639 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt @@ -153,7 +153,8 @@ add_subdirectory(main/native/libhdfs-examples) # Temporary fix to disable Libhdfs++ build on older systems that do not support thread_local include(CheckCXXSourceCompiles) unset (THREAD_LOCAL_SUPPORTED CACHE) -set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11") +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) check_cxx_source_compiles( "#include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 939747c34b444..914fefdafce6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -53,7 +53,8 @@ include(CheckCXXSourceCompiles) # Check if thread_local is supported unset (THREAD_LOCAL_SUPPORTED CACHE) -set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11") +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) check_cxx_source_compiles( "#include @@ -149,12 +150,13 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0") if(UNIX) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -std=c++11 -g -fPIC -fno-strict-aliasing") +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -g -fPIC -fno-strict-aliasing") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing") endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_STANDARD 11) add_definitions(-DASIO_HAS_STD_ADDRESSOF -DASIO_HAS_STD_ARRAY -DASIO_HAS_STD_ATOMIC -DASIO_HAS_CSTDINT -DASIO_HAS_STD_SHARED_PTR -DASIO_HAS_STD_TYPE_TRAITS -DASIO_HAS_VARIADIC_TEMPLATES -DASIO_HAS_STD_FUNCTION -DASIO_HAS_STD_CHRONO -DASIO_HAS_STD_SYSTEM_ERROR) endif () From 6c2ce3d56b14ce4139cbecc1cfe026e5a838f319 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Thu, 6 Aug 2020 19:15:16 +0530 Subject: [PATCH 064/335] YARN-10389. Option to override RMWebServices with custom WebService class Contributed by Tanu Ajmera. Reviewed by Bilwa ST and Sunil G. --- .../hadoop/yarn/conf/YarnConfiguration.java | 3 +++ .../src/main/resources/yarn-default.xml | 9 +++++++++ .../server/resourcemanager/webapp/RMWebApp.java | 17 ++++++++++------- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 19ce6221b01fe..6c1be0e34a018 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2406,6 +2406,9 @@ public static boolean isAclEnabled(Configuration conf) { public static final String YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES = "yarn.http.rmwebapp.custom.unwrapped.dao.classes"; + public static final String YARN_WEBAPP_CUSTOM_WEBSERVICE_CLASS = + "yarn.webapp.custom.webservice.class"; + /** * Whether or not users are allowed to request that Docker containers honor * the debug deletion delay. This is useful for troubleshooting Docker diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2208b00d8a38e..47f123918b6a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3376,6 +3376,15 @@ + + + Used to specify custom WebServices class to bind with RMWebApp overriding + the default RMWebServices. + + yarn.webapp.custom.webservice.class + + + The Node Label script to run. Script output Line starting with "NODE_PARTITION:" will be considered as Node Label Partition. In case of diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 5075d2505635c..3ed53f6dcc669 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; +import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; @@ -44,6 +45,7 @@ public class RMWebApp extends WebApp implements YarnWebParams { LoggerFactory.getLogger(RMWebApp.class.getName()); private final ResourceManager rm; private boolean standby = false; + private Configuration conf; public RMWebApp(ResourceManager rm) { this.rm = rm; @@ -51,15 +53,17 @@ public RMWebApp(ResourceManager rm) { @Override public void setup() { + conf = rm.getConfig(); bind(JAXBContextResolver.class); - bind(RMWebServices.class); + Class webService = conf.getClass( + YarnConfiguration.YARN_WEBAPP_CUSTOM_WEBSERVICE_CLASS, + RMWebServices.class); + bind(webService); bind(GenericExceptionHandler.class); bind(RMWebApp.class).toInstance(this); bindExternalClasses(); + bind(ResourceManager.class).toInstance(rm); - if (rm != null) { - bind(ResourceManager.class).toInstance(rm); - } route("/", RmController.class); route(pajoin("/nodes", NODE_STATE), RmController.class, "nodes"); route(pajoin("/apps", APP_STATE), RmController.class); @@ -99,8 +103,7 @@ public String getRedirectPath() { } private void bindExternalClasses() { - YarnConfiguration yarnConf = new YarnConfiguration(rm.getConfig()); - Class[] externalClasses = yarnConf + Class[] externalClasses = conf .getClasses(YarnConfiguration.YARN_HTTP_WEBAPP_EXTERNAL_CLASSES); for (Class c : externalClasses) { bind(c); @@ -111,7 +114,7 @@ private void bindExternalClasses() { private String buildRedirectPath() { // make a copy of the original configuration so not to mutate it. Also use // an YarnConfiguration to force loading of yarn-site.xml. - YarnConfiguration yarnConf = new YarnConfiguration(rm.getConfig()); + YarnConfiguration yarnConf = new YarnConfiguration(conf); String activeRMHAId = RMHAUtils.findActiveRMHAId(yarnConf); String path = ""; if (activeRMHAId != null) { From 3fd3aeb621e9a958486fb6a21653855c4c66f31f Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 11 Aug 2020 08:52:16 -0700 Subject: [PATCH 065/335] HDFS-15492. Make trash root inside each snapshottable directory (#2176) --- .../apache/hadoop/fs/FsServerDefaults.java | 18 ++ .../org/apache/hadoop/hdfs/DFSClient.java | 26 ++ .../org/apache/hadoop/hdfs/DFSUtilClient.java | 12 + .../hadoop/hdfs/DistributedFileSystem.java | 110 ++++++- .../hdfs/protocolPB/PBHelperClient.java | 6 +- .../src/main/proto/hdfs.proto | 1 + .../hdfs/server/namenode/FSNamesystem.java | 10 +- .../hdfs/TestDistributedFileSystem.java | 298 ++++++++++++++++++ 8 files changed, 466 insertions(+), 15 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java index f6396482d18c1..43d2f3ef19a6c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java @@ -56,6 +56,7 @@ public Writable newInstance() { private DataChecksum.Type checksumType; private String keyProviderUri; private byte storagepolicyId; + private boolean snapshotTrashRootEnabled; public FsServerDefaults() { } @@ -83,6 +84,18 @@ public FsServerDefaults(long blockSize, int bytesPerChecksum, boolean encryptDataTransfer, long trashInterval, DataChecksum.Type checksumType, String keyProviderUri, byte storagepolicy) { + this(blockSize, bytesPerChecksum, writePacketSize, replication, + fileBufferSize, encryptDataTransfer, trashInterval, + checksumType, keyProviderUri, storagepolicy, + false); + } + + public FsServerDefaults(long blockSize, int bytesPerChecksum, + int writePacketSize, short replication, int fileBufferSize, + boolean encryptDataTransfer, long trashInterval, + DataChecksum.Type checksumType, + String keyProviderUri, byte storagepolicy, + boolean snapshotTrashRootEnabled) { this.blockSize = blockSize; this.bytesPerChecksum = bytesPerChecksum; this.writePacketSize = writePacketSize; @@ -93,6 +106,7 @@ public FsServerDefaults(long blockSize, int bytesPerChecksum, this.checksumType = checksumType; this.keyProviderUri = keyProviderUri; this.storagepolicyId = storagepolicy; + this.snapshotTrashRootEnabled = snapshotTrashRootEnabled; } public long getBlockSize() { @@ -139,6 +153,10 @@ public byte getDefaultStoragePolicyId() { return storagepolicyId; } + public boolean getSnapshotTrashRootEnabled() { + return snapshotTrashRootEnabled; + } + // ///////////////////////////////////////// // Writable // ///////////////////////////////////////// diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index e9b5dd2b0d208..0f7e1bb963496 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -3149,6 +3149,32 @@ boolean isHDFSEncryptionEnabled() throws IOException { return getKeyProviderUri() != null; } + boolean isSnapshotTrashRootEnabled() throws IOException { + return getServerDefaults().getSnapshotTrashRootEnabled(); + } + + /** + * Get the snapshot root of a given file or directory if it exists. + * e.g. if /snapdir1 is a snapshottable directory and path given is + * /snapdir1/path/to/file, this method would return /snapdir1 + * @param path Path to a file or a directory. + * @return Not null if found in a snapshot root directory. + * @throws IOException + */ + String getSnapshotRoot(Path path) throws IOException { + SnapshottableDirectoryStatus[] dirStatusList = getSnapshottableDirListing(); + if (dirStatusList == null) { + return null; + } + for (SnapshottableDirectoryStatus dirStatus : dirStatusList) { + String currDir = dirStatus.getFullPath().toString(); + if (path.toUri().getPath().startsWith(currDir)) { + return currDir; + } + } + return null; + } + /** * Returns the SaslDataTransferClient configured for this DFSClient. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 95aad12d928b2..b5f47a9a96226 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -1040,4 +1040,16 @@ public static String getEZTrashRoot(EncryptionZone ez, return (ezpath.equals("/") ? ezpath : ezpath + Path.SEPARATOR) + FileSystem.TRASH_PREFIX + Path.SEPARATOR + ugi.getShortUserName(); } + + /** + * Returns trash root in a snapshottable directory. + * @param ssRoot String of path to a snapshottable directory root. + * @param ugi user of trash owner. + * @return unqualified path of trash root. + */ + public static String getSnapshotTrashRoot(String ssRoot, + UserGroupInformation ugi) { + return (ssRoot.equals("/") ? ssRoot : ssRoot + Path.SEPARATOR) + + FileSystem.TRASH_PREFIX + Path.SEPARATOR + ugi.getShortUserName(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 37d0226a3a326..7694f789e1cf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -129,10 +129,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.Set; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -3273,8 +3275,11 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies( /** * Get the root directory of Trash for a path in HDFS. * 1. File in encryption zone returns /ez1/.Trash/username - * 2. File not in encryption zone, or encountered exception when checking - * the encryption zone of the path, returns /users/username/.Trash + * 2. File in snapshottable directory returns /snapdir1/.Trash/username + * if dfs.namenode.snapshot.trashroot.enabled is set to true. + * 3. In other cases, or encountered exception when checking the encryption + * zone or when checking snapshot root of the path, returns + * /users/username/.Trash * Caller appends either Current or checkpoint timestamp for trash destination * @param path the trash root of the path to be determined. * @return trash root @@ -3283,41 +3288,89 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies( public Path getTrashRoot(Path path) { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOT); + if (path == null) { + return super.getTrashRoot(null); + } + + // Snapshottable directory trash root, not null if path is inside a + // snapshottable directory and isSnapshotTrashRootEnabled is true from NN. + String ssTrashRoot = null; try { - if ((path == null) || !dfs.isHDFSEncryptionEnabled()) { - return super.getTrashRoot(path); + if (dfs.isSnapshotTrashRootEnabled()) { + String ssRoot = dfs.getSnapshotRoot(path); + if (ssRoot != null) { + ssTrashRoot = DFSUtilClient.getSnapshotTrashRoot(ssRoot, dfs.ugi); + } + } + } catch (IOException ioe) { + DFSClient.LOG.warn("Exception while checking whether the path is in a " + + "snapshottable directory", ioe); + } + + try { + if (!dfs.isHDFSEncryptionEnabled()) { + if (ssTrashRoot == null) { + // the path is not in a snapshottable directory and EZ is not enabled + return super.getTrashRoot(path); + } else { + return this.makeQualified(new Path(ssTrashRoot)); + } } } catch (IOException ioe) { DFSClient.LOG.warn("Exception while checking whether encryption zone is " + "supported", ioe); } - String parentSrc = path.isRoot()? - path.toUri().getPath():path.getParent().toUri().getPath(); + // HDFS encryption is enabled on the cluster at this point, does not + // necessary mean the given path is in an EZ hence the check. + String parentSrc = path.isRoot() ? + path.toUri().getPath() : path.getParent().toUri().getPath(); + String ezTrashRoot = null; try { EncryptionZone ez = dfs.getEZForPath(parentSrc); if ((ez != null)) { - return this.makeQualified( - new Path(DFSUtilClient.getEZTrashRoot(ez, dfs.ugi))); + ezTrashRoot = DFSUtilClient.getEZTrashRoot(ez, dfs.ugi); } } catch (IOException e) { DFSClient.LOG.warn("Exception in checking the encryption zone for the " + "path " + parentSrc + ". " + e.getMessage()); } - return super.getTrashRoot(path); + + if (ssTrashRoot == null) { + if (ezTrashRoot == null) { + // The path is neither in a snapshottable directory nor in an EZ + return super.getTrashRoot(path); + } else { + return this.makeQualified(new Path(ezTrashRoot)); + } + } else { + if (ezTrashRoot == null) { + return this.makeQualified(new Path(ssTrashRoot)); + } else { + // The path is in EZ and in a snapshottable directory + return this.makeQualified(new Path( + ssTrashRoot.length() > ezTrashRoot.length() ? + ssTrashRoot : ezTrashRoot)); + } + } } /** * Get all the trash roots of HDFS for current user or for all the users. - * 1. File deleted from non-encryption zone /user/username/.Trash - * 2. File deleted from encryption zones + * 1. File deleted from encryption zones * e.g., ez1 rooted at /ez1 has its trash root at /ez1/.Trash/$USER + * 2. File deleted from snapshottable directories + * if dfs.namenode.snapshot.trashroot.enabled is set to true. + * e.g., snapshottable directory /snapdir1 has its trash root + * at /snapdir1/.Trash/$USER + * 3. File deleted from other directories + * /user/username/.Trash * @param allUsers return trashRoots of all users if true, used by emptier * @return trash roots of HDFS */ @Override public Collection getTrashRoots(boolean allUsers) { - List ret = new ArrayList<>(); + Set ret = new HashSet<>(); // Get normal trash roots ret.addAll(super.getTrashRoots(allUsers)); @@ -3348,6 +3401,39 @@ public Collection getTrashRoots(boolean allUsers) { } catch (IOException e){ DFSClient.LOG.warn("Cannot get all encrypted trash roots", e); } + + try { + // Get snapshottable directory trash roots + if (dfs.isSnapshotTrashRootEnabled()) { + SnapshottableDirectoryStatus[] lst = dfs.getSnapshottableDirListing(); + if (lst != null) { + for (SnapshottableDirectoryStatus dirStatus : lst) { + String ssDir = dirStatus.getFullPath().toString(); + Path ssTrashRoot = new Path(ssDir, FileSystem.TRASH_PREFIX); + if (!exists(ssTrashRoot)) { + continue; + } + if (allUsers) { + for (FileStatus candidate : listStatus(ssTrashRoot)) { + if (exists(candidate.getPath())) { + ret.add(candidate); + } + } + } else { + Path userTrash = new Path(DFSUtilClient.getSnapshotTrashRoot( + ssDir, dfs.ugi)); + try { + ret.add(getFileStatus(userTrash)); + } catch (FileNotFoundException ignored) { + } + } + } + } + } + } catch (IOException e) { + DFSClient.LOG.warn("Cannot get snapshot trash roots", e); + } + return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index f3d0cd96ed129..c7b8f5f3078fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -2124,7 +2124,8 @@ public static FsServerDefaults convert(FsServerDefaultsProto fs) { fs.getTrashInterval(), convert(fs.getChecksumType()), fs.hasKeyProviderUri() ? fs.getKeyProviderUri() : null, - (byte) fs.getPolicyId()); + (byte) fs.getPolicyId(), + fs.getSnapshotTrashRootEnabled()); } public static List convert( @@ -2298,7 +2299,8 @@ public static FsServerDefaultsProto convert(FsServerDefaults fs) { .setEncryptDataTransfer(fs.getEncryptDataTransfer()) .setTrashInterval(fs.getTrashInterval()) .setChecksumType(convert(fs.getChecksumType())) - .setPolicyId(fs.getDefaultStoragePolicyId()); + .setPolicyId(fs.getDefaultStoragePolicyId()) + .setSnapshotTrashRootEnabled(fs.getSnapshotTrashRootEnabled()); if (fs.getKeyProviderUri() != null) { builder.setKeyProviderUri(fs.getKeyProviderUri()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 2440a7ac16d9a..7c49c2335c268 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -526,6 +526,7 @@ message FsServerDefaultsProto { optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32]; optional string keyProviderUri = 9; optional uint32 policyId = 10 [default = 0]; + optional bool snapshotTrashRootEnabled = 11 [default = false]; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index c0cbabb8401fd..dd590c6d80b4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -382,6 +382,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, public static final org.slf4j.Logger LOG = LoggerFactory .getLogger(FSNamesystem.class.getName()); + + // The following are private configurations + static final String DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED = + "dfs.namenode.snapshot.trashroot.enabled"; + static final boolean DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT = false; + private final MetricsRegistry registry = new MetricsRegistry("FSNamesystem"); @Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics = registry.newRatesWithAggregation("detailedLockHoldTimeMetrics"); @@ -902,7 +908,9 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { conf.getTrimmed( CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, ""), - blockManager.getStoragePolicySuite().getDefaultPolicy().getId()); + blockManager.getStoragePolicySuite().getDefaultPolicy().getId(), + conf.getBoolean(DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED, + DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT)); this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, DFS_NAMENODE_MAX_OBJECTS_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 29e49ea61519f..c4355d50fbbce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -42,6 +42,7 @@ import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; @@ -2144,4 +2145,301 @@ public void testECCloseCommittedBlock() throws Exception { LambdaTestUtils.intercept(IOException.class, "", () -> str.close()); } } + + @Test + public void testGetTrashRoot() throws IOException { + Configuration conf = getTestConfiguration(); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + try { + DistributedFileSystem dfs = cluster.getFileSystem(); + Path testDir = new Path("/ssgtr/test1/"); + Path file0path = new Path(testDir, "file-0"); + dfs.create(file0path); + + Path trBeforeAllowSnapshot = dfs.getTrashRoot(file0path); + String trBeforeAllowSnapshotStr = trBeforeAllowSnapshot.toUri().getPath(); + // The trash root should be in user home directory + String homeDirStr = dfs.getHomeDirectory().toUri().getPath(); + assertTrue(trBeforeAllowSnapshotStr.startsWith(homeDirStr)); + + dfs.allowSnapshot(testDir); + + Path trAfterAllowSnapshot = dfs.getTrashRoot(file0path); + String trAfterAllowSnapshotStr = trAfterAllowSnapshot.toUri().getPath(); + // The trash root should now be in the snapshot root + String testDirStr = testDir.toUri().getPath(); + assertTrue(trAfterAllowSnapshotStr.startsWith(testDirStr)); + + // Cleanup + dfs.disallowSnapshot(testDir); + dfs.delete(testDir, true); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + private boolean isPathInUserHome(String pathStr, DistributedFileSystem dfs) { + String homeDirStr = dfs.getHomeDirectory().toUri().getPath(); + return pathStr.startsWith(homeDirStr); + } + + @Test + public void testGetTrashRoots() throws IOException { + Configuration conf = getTestConfiguration(); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + try { + DistributedFileSystem dfs = cluster.getFileSystem(); + Path testDir = new Path("/ssgtr/test1/"); + Path file0path = new Path(testDir, "file-0"); + dfs.create(file0path); + // Create user trash + Path currUserHome = dfs.getHomeDirectory(); + Path currUserTrash = new Path(currUserHome, FileSystem.TRASH_PREFIX); + dfs.mkdirs(currUserTrash); + // Create trash inside test directory + Path testDirTrash = new Path(testDir, FileSystem.TRASH_PREFIX); + Path testDirTrashCurrUser = new Path(testDirTrash, + UserGroupInformation.getCurrentUser().getShortUserName()); + dfs.mkdirs(testDirTrashCurrUser); + + Collection trashRoots = dfs.getTrashRoots(false); + // getTrashRoots should only return 1 empty user trash in the home dir now + assertEquals(1, trashRoots.size()); + FileStatus firstFileStatus = trashRoots.iterator().next(); + String pathStr = firstFileStatus.getPath().toUri().getPath(); + assertTrue(isPathInUserHome(pathStr, dfs)); + // allUsers should not make a difference for now because we have one user + Collection trashRootsAllUsers = dfs.getTrashRoots(true); + assertEquals(trashRoots, trashRootsAllUsers); + + dfs.allowSnapshot(testDir); + + Collection trashRootsAfter = dfs.getTrashRoots(false); + // getTrashRoots should return 1 more trash root inside snapshottable dir + assertEquals(trashRoots.size() + 1, trashRootsAfter.size()); + boolean foundUserHomeTrash = false; + boolean foundSnapDirUserTrash = false; + String testDirStr = testDir.toUri().getPath(); + for (FileStatus fileStatus : trashRootsAfter) { + String currPathStr = fileStatus.getPath().toUri().getPath(); + if (isPathInUserHome(currPathStr, dfs)) { + foundUserHomeTrash = true; + } else if (currPathStr.startsWith(testDirStr)) { + foundSnapDirUserTrash = true; + } + } + assertTrue(foundUserHomeTrash); + assertTrue(foundSnapDirUserTrash); + // allUsers should not make a difference for now because we have one user + Collection trashRootsAfterAllUsers = dfs.getTrashRoots(true); + assertEquals(trashRootsAfter, trashRootsAfterAllUsers); + + // Create trash root for user0 + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user0"); + String user0HomeStr = DFSUtilClient.getHomeDirectory(conf, ugi); + Path user0Trash = new Path(user0HomeStr, FileSystem.TRASH_PREFIX); + dfs.mkdirs(user0Trash); + // allUsers flag set to false should be unaffected + Collection trashRootsAfter2 = dfs.getTrashRoots(false); + assertEquals(trashRootsAfter, trashRootsAfter2); + // allUsers flag set to true should include new user's trash + trashRootsAfter2 = dfs.getTrashRoots(true); + assertEquals(trashRootsAfter.size() + 1, trashRootsAfter2.size()); + + // Create trash root inside the snapshottable directory for user0 + Path testDirTrashUser0 = new Path(testDirTrash, ugi.getShortUserName()); + dfs.mkdirs(testDirTrashUser0); + Collection trashRootsAfter3 = dfs.getTrashRoots(true); + assertEquals(trashRootsAfter2.size() + 1, trashRootsAfter3.size()); + + // Cleanup + dfs.disallowSnapshot(testDir); + dfs.delete(testDir, true); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @Test + public void testGetTrashRootsOnSnapshottableDirWithEZ() + throws IOException, NoSuchAlgorithmException { + Configuration conf = getTestConfiguration(); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + // Set encryption zone config + File tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString()); + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + // Create key for EZ + final KeyProvider provider = + cluster.getNameNode().getNamesystem().getProvider(); + final KeyProvider.Options options = KeyProvider.options(conf); + provider.createKey("key", options); + provider.flush(); + + try { + DistributedFileSystem dfs = cluster.getFileSystem(); + Path testDir = new Path("/ssgtr/test2/"); + dfs.mkdirs(testDir); + dfs.createEncryptionZone(testDir, "key"); + + // Create trash inside test directory + Path testDirTrash = new Path(testDir, FileSystem.TRASH_PREFIX); + Path testDirTrashCurrUser = new Path(testDirTrash, + UserGroupInformation.getCurrentUser().getShortUserName()); + dfs.mkdirs(testDirTrashCurrUser); + + Collection trashRoots = dfs.getTrashRoots(false); + assertEquals(1, trashRoots.size()); + FileStatus firstFileStatus = trashRoots.iterator().next(); + String pathStr = firstFileStatus.getPath().toUri().getPath(); + String testDirStr = testDir.toUri().getPath(); + assertTrue(pathStr.startsWith(testDirStr)); + + dfs.allowSnapshot(testDir); + + Collection trashRootsAfter = dfs.getTrashRoots(false); + // getTrashRoots should give the same result + assertEquals(trashRoots, trashRootsAfter); + + // Cleanup + dfs.disallowSnapshot(testDir); + dfs.delete(testDir, true); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @Test + public void testGetTrashRootOnSnapshottableDirInEZ() + throws IOException, NoSuchAlgorithmException { + Configuration conf = getTestConfiguration(); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + // Set EZ config + File tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString()); + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + // Create key for EZ + final KeyProvider provider = + cluster.getNameNode().getNamesystem().getProvider(); + final KeyProvider.Options options = KeyProvider.options(conf); + provider.createKey("key", options); + provider.flush(); + + try { + DistributedFileSystem dfs = cluster.getFileSystem(); + + Path testDir = new Path("/ssgtr/test3ez/"); + dfs.mkdirs(testDir); + dfs.createEncryptionZone(testDir, "key"); + Path testSubD = new Path(testDir, "sssubdir"); + Path file1Path = new Path(testSubD, "file1"); + dfs.create(file1Path); + + final Path trBefore = dfs.getTrashRoot(file1Path); + final String trBeforeStr = trBefore.toUri().getPath(); + // The trash root should be directly under testDir + final Path testDirTrash = new Path(testDir, FileSystem.TRASH_PREFIX); + final String testDirTrashStr = testDirTrash.toUri().getPath(); + assertTrue(trBeforeStr.startsWith(testDirTrashStr)); + + dfs.allowSnapshot(testSubD); + final Path trAfter = dfs.getTrashRoot(file1Path); + final String trAfterStr = trAfter.toUri().getPath(); + // The trash is now located in the dir inside + final Path testSubDirTrash = new Path(testSubD, FileSystem.TRASH_PREFIX); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + final Path testSubDirUserTrash = new Path(testSubDirTrash, + ugi.getShortUserName()); + final String testSubDirUserTrashStr = + testSubDirUserTrash.toUri().getPath(); + assertEquals(testSubDirUserTrashStr, trAfterStr); + + // Cleanup + dfs.disallowSnapshot(testSubD); + dfs.delete(testDir, true); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @Test + public void testGetTrashRootOnEZInSnapshottableDir() + throws IOException, NoSuchAlgorithmException { + Configuration conf = getTestConfiguration(); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + // Set EZ config + File tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString()); + final Path jksPath = new Path(tmpDir.toString(), "test.jks"); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + // Create key for EZ + final KeyProvider provider = + cluster.getNameNode().getNamesystem().getProvider(); + final KeyProvider.Options options = KeyProvider.options(conf); + provider.createKey("key", options); + provider.flush(); + + try { + DistributedFileSystem dfs = cluster.getFileSystem(); + + Path testDir = new Path("/ssgtr/test3ss/"); + dfs.mkdirs(testDir); + dfs.allowSnapshot(testDir); + Path testSubD = new Path(testDir, "ezsubdir"); + dfs.mkdirs(testSubD); + Path file1Path = new Path(testSubD, "file1"); + dfs.create(file1Path); + + final Path trBefore = dfs.getTrashRoot(file1Path); + final String trBeforeStr = trBefore.toUri().getPath(); + // The trash root should be directly under testDir + final Path testDirTrash = new Path(testDir, FileSystem.TRASH_PREFIX); + final String testDirTrashStr = testDirTrash.toUri().getPath(); + assertTrue(trBeforeStr.startsWith(testDirTrashStr)); + + // Need to remove the file inside the dir to establish EZ + dfs.delete(file1Path, false); + dfs.createEncryptionZone(testSubD, "key"); + dfs.create(file1Path); + + final Path trAfter = dfs.getTrashRoot(file1Path); + final String trAfterStr = trAfter.toUri().getPath(); + // The trash is now located in the dir inside + final Path testSubDirTrash = new Path(testSubD, FileSystem.TRASH_PREFIX); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + final Path testSubDirUserTrash = new Path(testSubDirTrash, + ugi.getShortUserName()); + final String testSubDirUserTrashStr = + testSubDirUserTrash.toUri().getPath(); + assertEquals(testSubDirUserTrashStr, trAfterStr); + + // Cleanup + dfs.disallowSnapshot(testDir); + dfs.delete(testDir, true); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } From 11cec9ab94086f12b329fa641c4669db802adcbc Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Tue, 11 Aug 2020 21:41:28 -0700 Subject: [PATCH 066/335] HDFS-15523. Fix findbugs warnings from HDFS-15520. (#2218) --- .../visitor/NamespacePrintVisitor.java | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java index 0294e62bbdf38..dbd0d5541ad11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java @@ -32,10 +32,6 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; @@ -50,17 +46,10 @@ * \- file3 (INodeFile@78392d6) * \- z_file4 (INodeFile@45848712) */ -public class NamespacePrintVisitor implements NamespaceVisitor { +public final class NamespacePrintVisitor implements NamespaceVisitor { static final String NON_LAST_ITEM = "+-"; static final String LAST_ITEM = "\\-"; - /** Print the tree from the given root to a {@link File}. */ - public static void print2File(INode root, File f) throws IOException { - try(final PrintWriter out = new PrintWriter(new FileWriter(f), true)) { - new NamespacePrintVisitor(out).print(root); - } - } - /** @return string of the tree in the given {@link FSNamesystem}. */ public static String print2Sting(FSNamesystem ns) { return print2Sting(ns.getFSDirectory().getRoot()); @@ -73,14 +62,6 @@ public static String print2Sting(INode root) { return out.getBuffer().toString(); } - /** - * Print the tree in the given {@link FSNamesystem} - * to the given {@link PrintStream}. - */ - public static void print(FSNamesystem ns, PrintStream out) { - new NamespacePrintVisitor(new PrintWriter(out)).print(ns); - } - private final PrintWriter out; private final StringBuffer prefix = new StringBuffer(); @@ -88,10 +69,6 @@ private NamespacePrintVisitor(PrintWriter out) { this.out = out; } - private void print(FSNamesystem namesystem) { - print(namesystem.getFSDirectory().getRoot()); - } - private void print(INode root) { root.accept(this, Snapshot.CURRENT_STATE_ID); } From 141c62584bd253f0831d39557f54627f26cef7f7 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 12 Aug 2020 16:08:36 +0900 Subject: [PATCH 067/335] HADOOP-17204. Fix typo in Hadoop KMS document. Contributed by Xieming Li. --- hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm index 656e1cf534abf..95e926b3561a1 100644 --- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm +++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm @@ -716,7 +716,7 @@ $H4 HTTP Kerberos Principals Configuration When KMS instances are behind a load-balancer or VIP, clients will use the hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the URL is used to construct the Kerberos service name of the server, `HTTP/#HOSTNAME#`. This means that all KMS instances must have a Kerberos service name with the load-balancer or VIP hostname. -In order to be able to access directly a specific KMS instance, the KMS instance must also have Keberos service name with its own hostname. This is required for monitoring and admin purposes. +In order to be able to access directly a specific KMS instance, the KMS instance must also have Kerberos service name with its own hostname. This is required for monitoring and admin purposes. Both Kerberos service principal credentials (for the load-balancer/VIP hostname and for the actual KMS instance hostname) must be in the keytab file configured for authentication. And the principal name specified in the configuration must be '\*'. For example: From 10716040a859b7127a4e1781be21c3b0b59dd456 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 12 Aug 2020 09:02:47 +0100 Subject: [PATCH 068/335] HDFS-15493. Update block map and name cache in parallel while loading fsimage. Contributed by Chengwei Wang --- .../server/namenode/FSImageFormatPBINode.java | 119 ++++++++++++------ .../namenode/FSImageFormatProtobuf.java | 1 + .../hdfs/server/namenode/TestFSImage.java | 61 +++++++++ 3 files changed, 142 insertions(+), 39 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 6212e65e01db6..22bb9056d2430 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -28,8 +28,9 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -204,15 +205,20 @@ public static void updateBlocksMap(INodeFile file, BlockManager bm) { private final FSDirectory dir; private final FSNamesystem fsn; private final FSImageFormatProtobuf.Loader parent; - private ReentrantLock cacheNameMapLock; - private ReentrantLock blockMapLock; + + // Update blocks map by single thread asynchronously + private ExecutorService blocksMapUpdateExecutor; + // update name cache by single thread asynchronously. + private ExecutorService nameCacheUpdateExecutor; Loader(FSNamesystem fsn, final FSImageFormatProtobuf.Loader parent) { this.fsn = fsn; this.dir = fsn.dir; this.parent = parent; - cacheNameMapLock = new ReentrantLock(true); - blockMapLock = new ReentrantLock(true); + // Note: these executors must be SingleThreadExecutor, as they + // are used to modify structures which are not thread safe. + blocksMapUpdateExecutor = Executors.newSingleThreadExecutor(); + nameCacheUpdateExecutor = Executors.newSingleThreadExecutor(); } void loadINodeDirectorySectionInParallel(ExecutorService service, @@ -263,7 +269,6 @@ void loadINodeDirectorySectionInParallel(ExecutorService service, void loadINodeDirectorySection(InputStream in) throws IOException { final List refList = parent.getLoaderContext() .getRefList(); - ArrayList inodeList = new ArrayList<>(); while (true) { INodeDirectorySection.DirEntry e = INodeDirectorySection.DirEntry .parseDelimitedFrom(in); @@ -274,15 +279,7 @@ void loadINodeDirectorySection(InputStream in) throws IOException { INodeDirectory p = dir.getInode(e.getParent()).asDirectory(); for (long id : e.getChildrenList()) { INode child = dir.getInode(id); - if (addToParent(p, child)) { - if (child.isFile()) { - inodeList.add(child); - } - if (inodeList.size() >= DIRECTORY_ENTRY_BATCH_SIZE) { - addToCacheAndBlockMap(inodeList); - inodeList.clear(); - } - } else { + if (!addToParent(p, child)) { LOG.warn("Failed to add the inode {} to the directory {}", child.getId(), p.getId()); } @@ -290,40 +287,79 @@ void loadINodeDirectorySection(InputStream in) throws IOException { for (int refId : e.getRefChildrenList()) { INodeReference ref = refList.get(refId); - if (addToParent(p, ref)) { - if (ref.isFile()) { - inodeList.add(ref); - } - if (inodeList.size() >= DIRECTORY_ENTRY_BATCH_SIZE) { - addToCacheAndBlockMap(inodeList); - inodeList.clear(); - } - } else { + if (!addToParent(p, ref)) { LOG.warn("Failed to add the inode reference {} to the directory {}", ref.getId(), p.getId()); } } } - addToCacheAndBlockMap(inodeList); } - private void addToCacheAndBlockMap(ArrayList inodeList) { - try { - cacheNameMapLock.lock(); - for (INode i : inodeList) { - dir.cacheName(i); - } - } finally { - cacheNameMapLock.unlock(); + private void fillUpInodeList(ArrayList inodeList, INode inode) { + if (inode.isFile()) { + inodeList.add(inode); } + if (inodeList.size() >= DIRECTORY_ENTRY_BATCH_SIZE) { + addToCacheAndBlockMap(inodeList); + inodeList.clear(); + } + } - try { - blockMapLock.lock(); - for (INode i : inodeList) { - updateBlocksMap(i.asFile(), fsn.getBlockManager()); + private void addToCacheAndBlockMap(final ArrayList inodeList) { + final ArrayList inodes = new ArrayList<>(inodeList); + nameCacheUpdateExecutor.submit( + new Runnable() { + @Override + public void run() { + addToCacheInternal(inodes); + } + }); + blocksMapUpdateExecutor.submit( + new Runnable() { + @Override + public void run() { + updateBlockMapInternal(inodes); + } + }); + } + + // update name cache with non-thread safe + private void addToCacheInternal(ArrayList inodeList) { + for (INode i : inodeList) { + dir.cacheName(i); + } + } + + // update blocks map with non-thread safe + private void updateBlockMapInternal(ArrayList inodeList) { + for (INode i : inodeList) { + updateBlocksMap(i.asFile(), fsn.getBlockManager()); + } + } + + void waitBlocksMapAndNameCacheUpdateFinished() throws IOException { + long start = System.currentTimeMillis(); + waitExecutorTerminated(blocksMapUpdateExecutor); + waitExecutorTerminated(nameCacheUpdateExecutor); + LOG.info("Completed update blocks map and name cache, total waiting " + + "duration {}ms.", (System.currentTimeMillis() - start)); + } + + private void waitExecutorTerminated(ExecutorService executorService) + throws IOException { + executorService.shutdown(); + long start = System.currentTimeMillis(); + while (!executorService.isTerminated()) { + try { + executorService.awaitTermination(1, TimeUnit.SECONDS); + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to executor service terminated duration {}ms.", + (System.currentTimeMillis() - start)); + } + } catch (InterruptedException e) { + LOG.error("Interrupted waiting for executor terminated.", e); + throw new IOException(e); } - } finally { - blockMapLock.unlock(); } } @@ -340,6 +376,7 @@ private int loadINodesInSection(InputStream in, Counter counter) // As the input stream is a LimitInputStream, the reading will stop when // EOF is encountered at the end of the stream. int cntr = 0; + ArrayList inodeList = new ArrayList<>(); while (true) { INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); if (p == null) { @@ -354,12 +391,16 @@ private int loadINodesInSection(InputStream in, Counter counter) synchronized(this) { dir.addToInodeMap(n); } + fillUpInodeList(inodeList, n); } cntr++; if (counter != null) { counter.increment(); } } + if (inodeList.size() > 0){ + addToCacheAndBlockMap(inodeList); + } return cntr; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java index be21d1f80f9e9..d34da5f6bea72 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -447,6 +447,7 @@ public int compare(FileSummary.Section s1, FileSummary.Section s2) { } else { inodeLoader.loadINodeDirectorySection(in); } + inodeLoader.waitBlocksMapAndNameCacheUpdateFinished(); break; case FILES_UNDERCONSTRUCTION: inodeLoader.loadFilesUnderConstructionSection(in); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 793a749be21c4..39a0f15c8b519 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -36,6 +36,7 @@ import java.util.EnumSet; import com.google.common.collect.Lists; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.Block; @@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.NativeCodeLoader; @@ -1152,4 +1154,63 @@ private void ensureSubSectionsAlignWithParent(ArrayList
subSec, // The first sub-section and parent section should have the same offset assertEquals(parent.getOffset(), subSec.get(0).getOffset()); } + + @Test + public void testUpdateBlocksMapAndNameCacheAsync() throws IOException { + Configuration conf = new Configuration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + FSDirectory fsdir = cluster.getNameNode().namesystem.getFSDirectory(); + File workingDir = GenericTestUtils.getTestDir(); + + File preRestartTree = new File(workingDir, "preRestartTree"); + File postRestartTree = new File(workingDir, "postRestartTree"); + + Path baseDir = new Path("/user/foo"); + fs.mkdirs(baseDir); + fs.allowSnapshot(baseDir); + for (int i = 0; i < 5; i++) { + Path dir = new Path(baseDir, Integer.toString(i)); + fs.mkdirs(dir); + for (int j = 0; j < 5; j++) { + Path file = new Path(dir, Integer.toString(j)); + FSDataOutputStream os = fs.create(file); + os.write((byte) j); + os.close(); + } + fs.createSnapshot(baseDir, "snap_"+i); + fs.rename(new Path(dir, "0"), new Path(dir, "renamed")); + } + SnapshotTestHelper.dumpTree2File(fsdir, preRestartTree); + + // checkpoint + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.restartNameNode(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fsdir = cluster.getNameNode().namesystem.getFSDirectory(); + + // Ensure all the files created above exist, and blocks is correct. + for (int i = 0; i < 5; i++) { + Path dir = new Path(baseDir, Integer.toString(i)); + assertTrue(fs.getFileStatus(dir).isDirectory()); + for (int j = 0; j < 5; j++) { + Path file = new Path(dir, Integer.toString(j)); + if (j == 0) { + file = new Path(dir, "renamed"); + } + FSDataInputStream in = fs.open(file); + int n = in.readByte(); + assertEquals(j, n); + in.close(); + } + } + SnapshotTestHelper.dumpTree2File(fsdir, postRestartTree); + SnapshotTestHelper.compareDumpedTreeInFile( + preRestartTree, postRestartTree, true); + } } \ No newline at end of file From e592ec5f8bf68ff15e910653f9e02a1bf4f4725c Mon Sep 17 00:00:00 2001 From: Aryan Gupta <44232823+aryangupta1998@users.noreply.github.com> Date: Thu, 13 Aug 2020 00:09:38 +0530 Subject: [PATCH 069/335] HDFS-15518. Wrong operation name in FsNamesystem for listSnapshots. (#2217) * HDFS-15518. Fixed String operationName = ListSnapshot. * HDFS-15518. Changed ListSnapshot to operationName. Co-authored-by: Aryan Gupta --- .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index dd590c6d80b4f..cd8e8ddb19bef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7031,7 +7031,7 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() */ public SnapshotStatus[] getSnapshotListing(String snapshotRoot) throws IOException { - final String operationName = "listSnapshotDirectory"; + final String operationName = "ListSnapshot"; SnapshotStatus[] status; checkOperation(OperationCategory.READ); boolean success = false; @@ -7048,10 +7048,10 @@ public SnapshotStatus[] getSnapshotListing(String snapshotRoot) readUnlock(operationName, getLockReportInfoSupplier(null)); } } catch (AccessControlException ace) { - logAuditEvent(success, "listSnapshots", snapshotRoot); + logAuditEvent(success, operationName, snapshotRoot); throw ace; } - logAuditEvent(success, "listSnapshots", snapshotRoot); + logAuditEvent(success, operationName, snapshotRoot); return status; } /** From 4a400d3193521fb9be18130aad415f0f58e036e8 Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Thu, 13 Aug 2020 18:51:49 +0530 Subject: [PATCH 070/335] HADOOP-17192. ITestS3AHugeFilesSSECDiskBlock failing (#2221) Contributed by Mukund Thakur --- .../hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java | 5 +++++ .../org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java index 2e5185bf55d9d..a8635ea3cd792 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java @@ -23,6 +23,9 @@ import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled; /** @@ -45,6 +48,8 @@ public void setup() throws Exception { @Override protected Configuration createScaleConfiguration() { Configuration conf = super.createScaleConfiguration(); + removeBaseAndBucketOverrides(conf, SERVER_SIDE_ENCRYPTION_KEY, + SERVER_SIDE_ENCRYPTION_ALGORITHM); S3ATestUtils.disableFilesystemCaching(conf); conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM, getSSEAlgorithm().getMethod()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java index eb80bc579f6c1..42b73d3d88c09 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java @@ -105,9 +105,7 @@ public void setup() throws Exception { * @return the configuration. */ private synchronized Configuration demandCreateConfiguration() { - if (conf == null) { - conf = createScaleConfiguration(); - } + conf = createScaleConfiguration(); return conf; } From cb50e3fcf70f8d5cb30238e96f87f9d6e2f2260a Mon Sep 17 00:00:00 2001 From: Vivek Ratnavel Subramanian Date: Thu, 13 Aug 2020 10:06:15 -0700 Subject: [PATCH 071/335] HDFS-15496. Add UI for deleted snapshots (#2212) --- .../hadoop/hdfs/protocol/SnapshotStatus.java | 54 ------------------- .../hadoop/hdfs/protocol/SnapshotInfo.java | 14 +++-- .../namenode/snapshot/SnapshotManager.java | 11 ++-- .../src/main/webapps/hdfs/dfshealth.html | 2 + 4 files changed, 19 insertions(+), 62 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java index 8c0dabd34ad42..3e2a7ae4453b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotStatus.java @@ -174,60 +174,6 @@ private static int maxLength(int n, Object value) { return Math.max(n, String.valueOf(value).length()); } - /** - * To be used to for collection of snapshot jmx. - */ - public static class Bean { - private final String path; - private final int snapshotID; - private final long modificationTime; - private final short permission; - private final String owner; - private final String group; - private final boolean isDeleted; - - - public Bean(String path, int snapshotID, long - modificationTime, short permission, String owner, String group, - boolean isDeleted) { - this.path = path; - this.snapshotID = snapshotID; - this.modificationTime = modificationTime; - this.permission = permission; - this.owner = owner; - this.group = group; - this.isDeleted = isDeleted; - } - - public String getPath() { - return path; - } - - public int getSnapshotID() { - return snapshotID; - } - - public long getModificationTime() { - return modificationTime; - } - - public short getPermission() { - return permission; - } - - public String getOwner() { - return owner; - } - - public String getGroup() { - return group; - } - - public boolean isDeleted() { - return isDeleted; - } - } - static String getSnapshotPath(String snapshottableDir, String snapshotRelativePath) { String parentFullPathStr = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java index 676e8276f258e..ef547788f1ac7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java @@ -82,18 +82,20 @@ public String toString() { } public static class Bean { - private final String snapshotID; + private final int snapshotID; private final String snapshotDirectory; private final long modificationTime; + private final String status; - public Bean(String snapshotID, String snapshotDirectory, - long modificationTime) { + public Bean(int snapshotID, String snapshotDirectory, + long modificationTime, boolean isMarkedAsDeleted) { this.snapshotID = snapshotID; this.snapshotDirectory = snapshotDirectory; this.modificationTime = modificationTime; + this.status = isMarkedAsDeleted ? "DELETED" : "ACTIVE"; } - public String getSnapshotID() { + public int getSnapshotID() { return snapshotID; } @@ -104,5 +106,9 @@ public String getSnapshotDirectory() { public long getModificationTime() { return modificationTime; } + + public String getStatus() { + return status; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 7569fc64e6666..3866125503325 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -746,16 +746,19 @@ public static SnapshottableDirectoryStatus.Bean toBean(INodeDirectory d) { d.getDirectorySnapshottableFeature().getNumSnapshots(), d.getDirectorySnapshottableFeature().getSnapshotQuota(), d.getModificationTime(), - Short.valueOf(Integer.toOctalString( - d.getFsPermissionShort())), + Short.parseShort(Integer.toOctalString(d.getFsPermissionShort())), d.getUserName(), d.getGroupName()); } public static SnapshotInfo.Bean toBean(Snapshot s) { + Snapshot.Root dir = s.getRoot(); return new SnapshotInfo.Bean( - s.getRoot().getLocalName(), s.getRoot().getFullPathName(), - s.getRoot().getModificationTime()); + s.getId(), + dir.getFullPathName(), + dir.getModificationTime(), + dir.isMarkedAsDeleted() + ); } private List getSnapshottableDirsForGc() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index fcf4e0d058613..8b03185d3d1d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -287,6 +287,7 @@ Snapshot ID Snapshot Directory Modification Time + Status {#Snapshots} @@ -294,6 +295,7 @@ {snapshotID} {snapshotDirectory} {modificationTime|date_tostring} + {status} {/Snapshots} From 86bbd38c8dc2aa4a7a5a7ac831c4c966c6eb231c Mon Sep 17 00:00:00 2001 From: hemanthboyina Date: Thu, 13 Aug 2020 23:44:42 +0530 Subject: [PATCH 072/335] YARN-10336. RM page should throw exception when command injected in RM REST API to get applications. Contributed by Bilwa S T. --- .../hadoop/yarn/server/webapp/AppsBlock.java | 3 +- .../yarn/server/webapp/TestAppsBlock.java | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestAppsBlock.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 6737e4d8fa9f8..40bd7024cf3a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -136,8 +136,7 @@ public void render(Block html) { try { fetchData(); - } - catch( Exception e) { + } catch (YarnException | IOException | InterruptedException e) { String message = "Failed to read the applications."; LOG.error(message, e); html.p().__(message).__(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestAppsBlock.java new file mode 100644 index 0000000000000..6853558130029 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestAppsBlock.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.webapp; + +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.view.BlockForTest; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.HtmlBlockForTest; +import org.junit.Test; + +public class TestAppsBlock { + + /** + * Test invalid application state.Exception should be thrown if application + * state is not valid. + */ + @Test(expected = IllegalArgumentException.class) + public void testInvalidAppState() { + AppsBlock appBlock = new AppsBlock(null, null) { + // override this so that apps block can fetch app state. + @Override + public Map moreParams() { + Map map = new HashMap<>(); + map.put(YarnWebParams.APP_STATE, "ACCEPTEDPING"); + return map; + } + + @Override + protected void renderData(Block html) { + } + }; + + // set up the test block to render AppsBlock + OutputStream outputStream = new ByteArrayOutputStream(); + HtmlBlock.Block block = createBlockToCreateTo(outputStream); + + // If application state is invalid it should throw exception + // instead of catching it. + appBlock.render(block); + } + + private static HtmlBlock.Block createBlockToCreateTo( + OutputStream outputStream) { + PrintWriter printWriter = new PrintWriter(outputStream); + HtmlBlock html = new HtmlBlockForTest(); + return new BlockForTest(html, printWriter, 10, false) { + @Override + protected void subView(Class cls) { + } + }; + }; + +} From aee3b97d093e1c63d96080506ea982136d73b1ed Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Fri, 14 Aug 2020 20:44:18 +0900 Subject: [PATCH 073/335] HADOOP-17206. Add python2 to required package on CentOS 8 for building documentation. (#2227) --- BUILDING.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BUILDING.txt b/BUILDING.txt index 78843ccfb95b4..9fc68e09740a2 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -436,6 +436,9 @@ Building on CentOS 8 $ sudo dnf group install --with-optional 'Development Tools' $ sudo dnf install java-1.8.0-openjdk-devel maven +* Install python2 for building documentation. + $ sudo dnf install python2 + * Install Protocol Buffers v3.7.1. $ git clone https://github.com/protocolbuffers/protobuf $ cd protobuf From 15a76e8d65690f2889a80e17b7a3216e3bae72a2 Mon Sep 17 00:00:00 2001 From: bshashikant Date: Fri, 14 Aug 2020 20:46:48 +0530 Subject: [PATCH 074/335] HDFS-15524. Add edit log entry for Snapshot deletion GC thread snapshot deletion. (#2219) --- .../hdfs/server/namenode/FSDirSnapshotOp.java | 10 ++-- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../TestOrderedSnapshotDeletionGc.java | 53 +++++++++++++++++-- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java index f264dc34063f1..d45c0c30d4907 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java @@ -269,15 +269,15 @@ static INode.BlocksMapUpdateInfo deleteSnapshot( // time of snapshot deletion final long now = Time.now(); final INode.BlocksMapUpdateInfo collectedBlocks = deleteSnapshot( - fsd, snapshotManager, iip, snapshotName, now); - fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName, - logRetryCache, now); + fsd, snapshotManager, iip, snapshotName, now, snapshotRoot, + logRetryCache); return collectedBlocks; } static INode.BlocksMapUpdateInfo deleteSnapshot( FSDirectory fsd, SnapshotManager snapshotManager, INodesInPath iip, - String snapshotName, long now) throws IOException { + String snapshotName, long now, String snapshotRoot, boolean logRetryCache) + throws IOException { INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo(); ChunkedArrayList removedINodes = new ChunkedArrayList<>(); INode.ReclaimContext context = new INode.ReclaimContext( @@ -293,6 +293,8 @@ static INode.BlocksMapUpdateInfo deleteSnapshot( fsd.writeUnlock(); } removedINodes.clear(); + fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName, + logRetryCache, now); return collectedBlocks; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index cd8e8ddb19bef..0a273f0a6d1d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7235,7 +7235,7 @@ public void gcDeletedSnapshot(String snapshotRoot, String snapshotName) final INodesInPath iip = dir.resolvePath(null, snapshotRoot, DirOp.WRITE); snapshotManager.assertMarkedAsDeleted(iip, snapshotName); blocksToBeDeleted = FSDirSnapshotOp.deleteSnapshot( - dir, snapshotManager, iip, snapshotName, now); + dir, snapshotManager, iip, snapshotName, now, snapshotRoot, false); } finally { writeUnlock(operationName, getLockReportInfoSupplier(rootPath)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java index b35134ba7b051..51c2efe886a2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOrderedSnapshotDeletionGc.java @@ -21,6 +21,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.hdfs.server.namenode.FSImage; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NNStorage; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; +import org.apache.hadoop.hdfs.util.Holder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; @@ -28,26 +35,30 @@ import org.junit.Test; import org.slf4j.event.Level; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Random; +import java.util.EnumMap; +import java.util.ArrayList; import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS; import static org.apache.hadoop.hdfs.server.namenode.snapshot.TestOrderedSnapshotDeletion.assertMarkedAsDeleted; import static org.apache.hadoop.hdfs.server.namenode.snapshot.TestOrderedSnapshotDeletion.assertNotMarkedAsDeleted; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Test {@link SnapshotDeletionGc}. */ public class TestOrderedSnapshotDeletionGc { private static final int GC_PERIOD = 10; - + private static final int NUM_DATANODES = 0; private MiniDFSCluster cluster; @Before @@ -56,7 +67,8 @@ public void setUp() throws Exception { conf.setBoolean(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, true); conf.setInt(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS, GC_PERIOD); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES) + .build(); cluster.waitActive(); GenericTestUtils.setLogLevel(SnapshotDeletionGc.LOG, Level.TRACE); @@ -117,6 +129,38 @@ public void testSingleDir() throws Exception { Assert.assertFalse(exist(s0path, hdfs)); waitForGc(Arrays.asList(s1path, s2path), hdfs); + // total no of edit log records created for delete snapshot will be equal + // to sum of no of user deleted snapshots and no of snapshots gc'ed with + // snapshotDeletion gc thread + doEditLogValidation(cluster, 5); + } + + static void doEditLogValidation(MiniDFSCluster cluster, + int editLogOpCount) throws Exception { + final FSNamesystem namesystem = cluster.getNamesystem(); + Configuration conf = cluster.getNameNode().getConf(); + FSImage fsimage = namesystem.getFSImage(); + Storage.StorageDirectory sd = fsimage.getStorage(). + dirIterator(NNStorage.NameNodeDirType.EDITS).next(); + cluster.shutdown(); + + File editFile = FSImageTestUtil.findLatestEditsLog(sd).getFile(); + assertTrue("Should exist: " + editFile, editFile.exists()); + EnumMap> counts; + counts = FSImageTestUtil.countEditLogOpTypes(editFile); + if (editLogOpCount > 0) { + assertEquals(editLogOpCount, (int) counts.get(FSEditLogOpCodes. + OP_DELETE_SNAPSHOT).held); + } + // make sure the gc thread doesn't start for a long time after the restart + conf.setInt(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS, + (int)(24 * 60_000L)); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES) + .build(); + cluster.waitActive(); + // ensure after the edits get replayed , all the snapshots are deleted + Assert.assertEquals(0, + cluster.getNamesystem().getSnapshotManager().getNumSnapshots()); } static boolean exist(Path snapshotRoot, DistributedFileSystem hdfs) @@ -168,6 +212,9 @@ public void testMultipleDirs() throws Exception { } waitForGc(snapshotPaths, hdfs); + // don't do edit log count validation here as gc snapshot + // deletion count will be random here + doEditLogValidation(cluster, -1); } static void createSnapshots(Path snapshottableDir, int numSnapshots, From b93dd7c281c176b4a4dd00f2a120ec2c2e2a60d4 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Fri, 14 Aug 2020 10:10:01 -0700 Subject: [PATCH 075/335] HDFS-15519. Check inaccessible INodes in FsImageValidation. (#2224) --- .../server/namenode/FsImageValidation.java | 154 ++++++++++++++++-- .../namenode/INodeReferenceValidation.java | 20 +-- .../namenode/visitor/INodeCountVisitor.java | 107 ++++++++++++ .../namenode/TestFsImageValidation.java | 6 +- 4 files changed, 263 insertions(+), 24 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/INodeCountVisitor.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java index 5dcb5069e870c..3b8c33165395b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java @@ -28,8 +28,12 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics; +import org.apache.hadoop.hdfs.server.namenode.visitor.INodeCountVisitor; +import org.apache.hadoop.hdfs.server.namenode.visitor.INodeCountVisitor.Counts; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.StringUtils; @@ -40,15 +44,21 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY; +import static org.apache.hadoop.hdfs.server.namenode.FsImageValidation.Cli.println; import static org.apache.hadoop.util.Time.now; /** @@ -134,6 +144,25 @@ static String toCommaSeparatedNumber(long n) { } return b.insert(0, n).toString(); } + + /** @return a filter for the given type. */ + static FilenameFilter newFilenameFilter(NameNodeFile type) { + final String prefix = type.getName() + "_"; + return new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (!name.startsWith(prefix)) { + return false; + } + for (int i = prefix.length(); i < name.length(); i++) { + if (!Character.isDigit(name.charAt(i))) { + return false; + } + } + return true; + } + }; + } } private final File fsImageFile; @@ -142,21 +171,44 @@ static String toCommaSeparatedNumber(long n) { this.fsImageFile = fsImageFile; } - int checkINodeReference(Configuration conf) throws Exception { + int run() throws Exception { + return run(new Configuration(), new AtomicInteger()); + } + + int run(AtomicInteger errorCount) throws Exception { + return run(new Configuration(), errorCount); + } + + int run(Configuration conf, AtomicInteger errorCount) throws Exception { + final int initCount = errorCount.get(); LOG.info(Util.memoryInfo()); initConf(conf); + // check INodeReference + final FSNamesystem namesystem = checkINodeReference(conf, errorCount); + + // check INodeMap + INodeMapValidation.run(namesystem.getFSDirectory(), errorCount); + LOG.info(Util.memoryInfo()); + + final int d = errorCount.get() - initCount; + if (d > 0) { + Cli.println("Found %d error(s) in %s", d, fsImageFile.getAbsolutePath()); + } + return d; + } + + private FSNamesystem loadImage(Configuration conf) throws IOException { final TimerTask checkProgress = new TimerTask() { @Override public void run() { final double percent = NameNode.getStartupProgress().createView() .getPercentComplete(Phase.LOADING_FSIMAGE); - LOG.info(String.format("%s Progress: %.1f%%", - Phase.LOADING_FSIMAGE, 100*percent)); + LOG.info(String.format("%s Progress: %.1f%% (%s)", + Phase.LOADING_FSIMAGE, 100*percent, Util.memoryInfo())); } }; - INodeReferenceValidation.start(); final Timer t = new Timer(); t.scheduleAtFixedRate(checkProgress, 0, 60_000); final long loadStart = now(); @@ -197,10 +249,42 @@ public void run() { t.cancel(); Cli.println("Loaded %s %s successfully in %s", FS_IMAGE, fsImageFile, StringUtils.formatTime(now() - loadStart)); + return namesystem; + } + + FSNamesystem checkINodeReference(Configuration conf, + AtomicInteger errorCount) throws Exception { + INodeReferenceValidation.start(); + final FSNamesystem namesystem = loadImage(conf); LOG.info(Util.memoryInfo()); - final int errorCount = INodeReferenceValidation.end(); + INodeReferenceValidation.end(errorCount); LOG.info(Util.memoryInfo()); - return errorCount; + return namesystem; + } + + static class INodeMapValidation { + static Iterable iterate(INodeMap map) { + return new Iterable() { + @Override + public Iterator iterator() { + return map.getMapIterator(); + } + }; + } + + static void run(FSDirectory fsdir, AtomicInteger errorCount) { + final int initErrorCount = errorCount.get(); + final Counts counts = INodeCountVisitor.countTree(fsdir.getRoot()); + for (INodeWithAdditionalFields i : iterate(fsdir.getINodeMap())) { + if (counts.getCount(i) == 0) { + Cli.printError(errorCount, "%s (%d) is inaccessible (%s)", + i, i.getId(), i.getFullPathName()); + } + } + println("%s ended successfully: %d error(s) found.", + INodeMapValidation.class.getSimpleName(), + errorCount.get() - initErrorCount); + } } static class Cli extends Configured implements Tool { @@ -217,9 +301,10 @@ public int run(String[] args) throws Exception { initLogLevels(); final FsImageValidation validation = FsImageValidation.newInstance(args); - final int errorCount = validation.checkINodeReference(getConf()); + final AtomicInteger errorCount = new AtomicInteger(); + validation.run(getConf(), errorCount); println("Error Count: %s", errorCount); - return errorCount == 0? 0: 1; + return errorCount.get() == 0? 0: 1; } static String parse(String... args) { @@ -240,19 +325,68 @@ static String parse(String... args) { return f; } - static void println(String format, Object... args) { + static synchronized void println(String format, Object... args) { final String s = String.format(format, args); System.out.println(s); LOG.info(s); } - static void printError(String message, Throwable t) { + static synchronized void warn(String format, Object... args) { + final String s = "WARN: " + String.format(format, args); + System.out.println(s); + LOG.warn(s); + } + + static synchronized void printError(String message, Throwable t) { System.out.println(message); if (t != null) { t.printStackTrace(System.out); } LOG.error(message, t); } + + static synchronized void printError(AtomicInteger errorCount, + String format, Object... args) { + final int count = errorCount.incrementAndGet(); + final String s = "FSIMAGE_ERROR " + count + ": " + + String.format(format, args); + System.out.println(s); + LOG.info(s); + } + } + + public static int validate(FSNamesystem namesystem) throws Exception { + final AtomicInteger errorCount = new AtomicInteger(); + final NNStorage nnStorage = namesystem.getFSImage().getStorage(); + for(Storage.StorageDirectory sd : nnStorage.getStorageDirs()) { + validate(sd.getCurrentDir(), errorCount); + } + return errorCount.get(); + } + + public static void validate(File path, AtomicInteger errorCount) + throws Exception { + if (path.isFile()) { + new FsImageValidation(path).run(errorCount); + } else if (path.isDirectory()) { + final File[] images = path.listFiles( + Util.newFilenameFilter(NameNodeFile.IMAGE)); + if (images == null || images.length == 0) { + Cli.warn("%s not found in %s", FSImage.class.getSimpleName(), + path.getAbsolutePath()); + return; + } + + Arrays.sort(images, Collections.reverseOrder()); + for (int i = 0; i < images.length; i++) { + final File image = images[i]; + Cli.println("%s %d) %s", FSImage.class.getSimpleName(), + i, image.getAbsolutePath()); + FsImageValidation.validate(image, errorCount); + } + } + + Cli.warn("%s is neither a file nor a directory", path.getAbsolutePath()); } public static void main(String[] args) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java index d3faf43074161..9c3190a82d886 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java @@ -46,18 +46,20 @@ public class INodeReferenceValidation { public static void start() { INSTANCE.compareAndSet(null, new INodeReferenceValidation()); - println("Validation started"); + println("%s started", INodeReferenceValidation.class.getSimpleName()); } - public static int end() { + public static void end(AtomicInteger errorCount) { final INodeReferenceValidation instance = INSTANCE.getAndSet(null); if (instance == null) { - return 0; + return; } - final int errorCount = instance.assertReferences(); - println("Validation ended successfully: %d error(s) found.", errorCount); - return errorCount; + final int initCount = errorCount.get(); + instance.assertReferences(errorCount); + println("%s ended successfully: %d error(s) found.", + INodeReferenceValidation.class.getSimpleName(), + errorCount.get() - initCount); } static void add(REF ref, Class clazz) { @@ -153,7 +155,7 @@ ReferenceSet getReferences( throw new IllegalArgumentException("References not found for " + clazz); } - private int assertReferences() { + private void assertReferences(AtomicInteger errorCount) { final int p = Runtime.getRuntime().availableProcessors(); LOG.info("Available Processors: {}", p); final ExecutorService service = Executors.newFixedThreadPool(p); @@ -168,7 +170,6 @@ public void run() { final Timer t = new Timer(); t.scheduleAtFixedRate(checkProgress, 0, 1_000); - final AtomicInteger errorCount = new AtomicInteger(); try { dstReferences.submit(errorCount, service); withCounts.submit(errorCount, service); @@ -183,7 +184,6 @@ public void run() { service.shutdown(); t.cancel(); } - return errorCount.get(); } static List> createTasks( @@ -215,7 +215,7 @@ public Integer call() throws Exception { try { ref.assertReferences(); } catch (Throwable t) { - println("%d: %s", errorCount.incrementAndGet(), t); + printError(errorCount, "%s", t); } } return references.size(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/INodeCountVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/INodeCountVisitor.java new file mode 100644 index 0000000000000..a3c936547851d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/INodeCountVisitor.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.visitor; + +import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * For validating {@link org.apache.hadoop.hdfs.server.namenode.FSImage}s. + */ +public class INodeCountVisitor implements NamespaceVisitor { + public interface Counts { + int getCount(INode inode); + } + + public static Counts countTree(INode root) { + return new INodeCountVisitor().count(root); + } + + private static class SetElement { + private final INode inode; + private final AtomicInteger count = new AtomicInteger(); + + SetElement(INode inode) { + this.inode = inode; + } + + int getCount() { + return count.get(); + } + + int incrementAndGet() { + return count.incrementAndGet(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } else if (obj == null || getClass() != obj.getClass()) { + return false; + } + final SetElement that = (SetElement) obj; + return this.inode.getId() == that.inode.getId(); + } + + @Override + public int hashCode() { + return Long.hashCode(inode.getId()); + } + } + + static class INodeSet implements Counts { + private final ConcurrentMap map + = new ConcurrentHashMap<>(); + + int put(INode inode, int snapshot) { + final SetElement key = new SetElement(inode); + final SetElement previous = map.putIfAbsent(key, key); + final SetElement current = previous != null? previous: key; + return current.incrementAndGet(); + } + + @Override + public int getCount(INode inode) { + final SetElement key = new SetElement(inode); + final SetElement value = map.get(key); + return value != null? value.getCount(): 0; + } + } + + private final INodeSet inodes = new INodeSet(); + + @Override + public INodeVisitor getDefaultVisitor() { + return new INodeVisitor() { + @Override + public void visit(INode iNode, int snapshot) { + inodes.put(iNode, snapshot); + } + }; + } + + private Counts count(INode root) { + root.accept(this, Snapshot.CURRENT_STATE_ID); + return inodes; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsImageValidation.java index 97bb2b9a33f9f..09f686ea59be5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsImageValidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsImageValidation.java @@ -43,13 +43,11 @@ public class TestFsImageValidation { * by the environment variable FS_IMAGE_FILE. */ @Test - public void testINodeReference() throws Exception { + public void testValidation() throws Exception { FsImageValidation.initLogLevels(); try { - final Configuration conf = new Configuration(); - final FsImageValidation validation = FsImageValidation.newInstance(); - final int errorCount = validation.checkINodeReference(conf); + final int errorCount = FsImageValidation.newInstance().run(); Assert.assertEquals("Error Count: " + errorCount, 0, errorCount); } catch (HadoopIllegalArgumentException e) { LOG.warn("The environment variable {} is not set: {}", From e3d1966f58ad473b8e852aa2b11c8ed2b434d9e4 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Sat, 15 Aug 2020 15:07:08 +0530 Subject: [PATCH 076/335] HDFS-15439. Setting dfs.mover.retry.max.attempts to negative value will retry forever. Contributed by AMC-team. --- .../org/apache/hadoop/hdfs/server/mover/Mover.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index 06c7cc5a5acf7..c63231d42205f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -134,9 +134,17 @@ private List getTargetStorages(StorageType t) { final int maxNoMoveInterval = conf.getInt( DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY, DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT); - this.retryMaxAttempts = conf.getInt( + final int maxAttempts = conf.getInt( DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT); + if (maxAttempts >= 0) { + this.retryMaxAttempts = maxAttempts; + } else { + LOG.warn(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY + " is " + + "configured with a negative value, using default value of " + + DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT); + this.retryMaxAttempts = DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT; + } this.retryCount = retryCount; this.dispatcher = new Dispatcher(nnc, Collections. emptySet(), Collections. emptySet(), movedWinWidth, moverThreads, 0, From 5092ea62ecbac840d56978a31bb11cfc14c6fe83 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Sat, 15 Aug 2020 12:51:08 +0100 Subject: [PATCH 077/335] HADOOP-13230. S3A to optionally retain directory markers. This adds an option to disable "empty directory" marker deletion, so avoid throttling and other scale problems. This feature is *not* backwards compatible. Consult the documentation and use with care. Contributed by Steve Loughran. Change-Id: I69a61e7584dc36e485d5e39ff25b1e3e559a1958 --- .../apache/hadoop/test/AssertExtensions.java | 74 ++ hadoop-tools/hadoop-aws/pom.xml | 58 ++ .../org/apache/hadoop/fs/s3a/Constants.java | 88 ++ .../fs/s3a/InconsistentAmazonS3Client.java | 4 +- .../org/apache/hadoop/fs/s3a/Listing.java | 25 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 505 +++++++---- .../apache/hadoop/fs/s3a/S3ListRequest.java | 27 +- .../apache/hadoop/fs/s3a/S3ListResult.java | 113 ++- .../hadoop/fs/s3a/impl/DirMarkerTracker.java | 352 ++++++++ .../hadoop/fs/s3a/impl/DirectoryPolicy.java | 110 +++ .../fs/s3a/impl/DirectoryPolicyImpl.java | 212 +++++ .../hadoop/fs/s3a/impl/RenameOperation.java | 261 +++++- .../hadoop/fs/s3a/impl/StatusProbeEnum.java | 30 +- .../fs/s3a/s3guard/DirListingMetadata.java | 25 +- .../fs/s3a/s3guard/ITtlTimeProvider.java | 13 + .../fs/s3a/s3guard/NullMetadataStore.java | 6 +- .../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 68 +- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 122 ++- .../hadoop/fs/s3a/tools/MarkerTool.java | 723 +++++++++++++++ .../fs/s3a/tools/MarkerToolOperations.java | 91 ++ .../s3a/tools/MarkerToolOperationsImpl.java | 70 ++ .../hadoop/fs/s3a/tools/package-info.java | 27 + .../tools/hadoop-aws/directory_markers.md | 694 +++++++++++++++ .../site/markdown/tools/hadoop-aws/index.md | 19 +- .../site/markdown/tools/hadoop-aws/s3guard.md | 16 +- .../site/markdown/tools/hadoop-aws/testing.md | 43 + .../hadoop/fs/s3a/AbstractS3ATestBase.java | 45 +- .../hadoop/fs/s3a/ITestAuthoritativePath.java | 12 +- .../fs/s3a/ITestS3ABucketExistence.java | 6 +- .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 212 +++-- .../fs/s3a/ITestS3AFileOperationCost.java | 686 +++++---------- .../fs/s3a/ITestS3ARemoteFileChanged.java | 2 +- .../hadoop/fs/s3a/ITestS3GuardEmptyDirs.java | 15 +- .../fs/s3a/ITestS3GuardListConsistency.java | 54 +- .../s3a/ITestS3GuardOutOfBandOperations.java | 14 +- .../hadoop/fs/s3a/S3ATestConstants.java | 13 +- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 33 +- .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 24 +- .../s3a/auth/ITestRestrictedReadAccess.java | 72 +- .../s3a/impl/TestDirectoryMarkerPolicy.java | 163 ++++ .../s3a/performance/AbstractS3ACostTest.java | 637 ++++++++++++++ .../ITestDirectoryMarkerListing.java | 824 ++++++++++++++++++ .../s3a/performance/ITestS3ADeleteCost.java | 218 +++++ .../s3a/performance/ITestS3ARenameCost.java | 207 +++++ .../fs/s3a/performance/OperationCost.java | 201 +++++ .../performance/OperationCostValidator.java | 483 ++++++++++ .../s3guard/AbstractS3GuardToolTestBase.java | 60 +- .../ITestS3GuardDDBRootOperations.java | 13 +- .../fs/s3a/s3guard/S3GuardToolTestHelper.java | 61 +- .../s3a/s3guard/TestDirListingMetadata.java | 11 +- .../fs/s3a/tools/AbstractMarkerToolTest.java | 334 +++++++ .../hadoop/fs/s3a/tools/ITestMarkerTool.java | 533 +++++++++++ .../tools/ITestMarkerToolRootOperations.java | 70 ++ 53 files changed, 7850 insertions(+), 929 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AssertExtensions.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicyImpl.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/package-info.java create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestDirectoryMarkerPolicy.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerToolRootOperations.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AssertExtensions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AssertExtensions.java new file mode 100644 index 0000000000000..8c5e553f71ee3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/AssertExtensions.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.test; + +import java.util.concurrent.Callable; + +import org.assertj.core.description.Description; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Extra classes to work with AssertJ. + * These are kept separate from {@link LambdaTestUtils} so there's + * no requirement for AssertJ to be on the classpath in that broadly + * used class. + */ +public final class AssertExtensions { + + private static final Logger LOG = + LoggerFactory.getLogger(AssertExtensions.class); + + private AssertExtensions() { + } + + /** + * A description for AssertJ "describedAs" clauses which evaluates the + * lambda-expression only on failure. That must return a string + * or null/"" to be skipped. + * @param eval lambda expression to invoke + * @return a description for AssertJ + */ + public static Description dynamicDescription(Callable eval) { + return new DynamicDescription(eval); + } + + private static final class DynamicDescription extends Description { + private final Callable eval; + + private DynamicDescription(final Callable eval) { + this.eval = eval; + } + + @Override + public String value() { + try { + return eval.call(); + } catch (Exception e) { + LOG.warn("Failed to evaluate description: " + e); + LOG.debug("Evaluation failure", e); + // return null so that the description evaluation chain + // will skip this one + return null; + } + } + } + + +} diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 5cdaf26007f7a..af8983e2ebe92 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -52,6 +52,10 @@ 200000 + + false + + @@ -123,6 +127,9 @@ ${fs.s3a.scale.test.huge.filesize} ${fs.s3a.scale.test.huge.partitionsize} ${fs.s3a.scale.test.timeout} + + ${fs.s3a.directory.marker.retention} + ${fs.s3a.directory.marker.audit} @@ -163,6 +170,7 @@ ${fs.s3a.s3guard.test.enabled} ${fs.s3a.s3guard.test.authoritative} ${fs.s3a.s3guard.test.implementation} + ${fs.s3a.directory.marker.retention} ${test.integration.timeout} @@ -189,6 +197,8 @@ **/ITestDynamoDBMetadataStoreScale.java **/ITestTerasort*.java + + **/ITestMarkerToolRootOperations.java **/ITestS3GuardDDBRootOperations.java @@ -215,6 +225,9 @@ ${fs.s3a.s3guard.test.enabled} ${fs.s3a.s3guard.test.implementation} ${fs.s3a.s3guard.test.authoritative} + + ${fs.s3a.directory.marker.retention} + ${fs.s3a.directory.marker.audit} @@ -230,6 +243,10 @@ **/ITestTerasort*.java + + + **/ITestMarkerToolRootOperations.java **/ITestS3AContractRootDir.java **/ITestS3GuardDDBRootOperations.java @@ -269,6 +286,9 @@ ${fs.s3a.s3guard.test.enabled} ${fs.s3a.s3guard.test.implementation} ${fs.s3a.s3guard.test.authoritative} + + ${fs.s3a.directory.marker.retention} + ${fs.s3a.directory.marker.audit} ${fs.s3a.scale.test.timeout} @@ -332,6 +352,44 @@ + + + keep-markers + + + markers + keep + + + + keep + + + + delete-markers + + + markers + delete + + + + delete + + + + auth-markers + + + markers + authoritative + + + + authoritative + + + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 22a0b45f1c7a5..a1c1d969a8258 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -953,4 +953,92 @@ private Constants() { * Value: {@value} seconds. */ public static final int THREAD_POOL_SHUTDOWN_DELAY_SECONDS = 30; + + /** + * Policy for directory markers. + * This is a new feature of HADOOP-13230 which addresses + * some scale, performance and permissions issues -but + * at the risk of backwards compatibility. + */ + public static final String DIRECTORY_MARKER_POLICY = + "fs.s3a.directory.marker.retention"; + + /** + * Delete directory markers. This is the backwards compatible option. + * Value: {@value}. + */ + public static final String DIRECTORY_MARKER_POLICY_DELETE = + "delete"; + + /** + * Retain directory markers. + * Value: {@value}. + */ + public static final String DIRECTORY_MARKER_POLICY_KEEP = + "keep"; + + /** + * Retain directory markers in authoritative directory trees only. + * Value: {@value}. + */ + public static final String DIRECTORY_MARKER_POLICY_AUTHORITATIVE = + "authoritative"; + + /** + * Default retention policy: {@value}. + */ + public static final String DEFAULT_DIRECTORY_MARKER_POLICY = + DIRECTORY_MARKER_POLICY_DELETE; + + + /** + * {@code PathCapabilities} probe to verify that an S3A Filesystem + * has the changes needed to safely work with buckets where + * directoy markers have not been deleted. + * Value: {@value}. + */ + public static final String STORE_CAPABILITY_DIRECTORY_MARKER_AWARE + = "fs.s3a.capability.directory.marker.aware"; + + /** + * {@code PathCapabilities} probe to indicate that the filesystem + * keeps directory markers. + * Value: {@value}. + */ + public static final String STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP + = "fs.s3a.capability.directory.marker.policy.keep"; + + /** + * {@code PathCapabilities} probe to indicate that the filesystem + * deletes directory markers. + * Value: {@value}. + */ + public static final String STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE + = "fs.s3a.capability.directory.marker.policy.delete"; + + /** + * {@code PathCapabilities} probe to indicate that the filesystem + * keeps directory markers in authoritative paths only. + * Value: {@value}. + */ + public static final String + STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE = + "fs.s3a.capability.directory.marker.policy.authoritative"; + + /** + * {@code PathCapabilities} probe to indicate that a path + * keeps directory markers. + * Value: {@value}. + */ + public static final String STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP + = "fs.s3a.capability.directory.marker.action.keep"; + + /** + * {@code PathCapabilities} probe to indicate that a path + * deletes directory markers. + * Value: {@value}. + */ + public static final String STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE + = "fs.s3a.capability.directory.marker.action.delete"; + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java index 34c043be9cb73..4cb05ae9e6512 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java @@ -330,7 +330,9 @@ private boolean isDescendant(String parent, String child, boolean recursive) { } else { Path actualParentPath = new Path(child).getParent(); Path expectedParentPath = new Path(parent); - return actualParentPath.equals(expectedParentPath); + // children which are directory markers are excluded here + return actualParentPath.equals(expectedParentPath) + && !child.endsWith("/"); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index fcb492857e617..34129e0bf1a74 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -142,12 +142,27 @@ public FileStatusListingIterator createFileStatusListingIterator( Listing.FileStatusAcceptor acceptor, RemoteIterator providedStatus) throws IOException { return new FileStatusListingIterator( - new ObjectListingIterator(listPath, request), + createObjectListingIterator(listPath, request), filter, acceptor, providedStatus); } + /** + * Create an object listing iterator against a path, with a given + * list object request. + * @param listPath path of the listing + * @param request initial request to make + * @return the iterator + * @throws IOException IO Problems + */ + @Retries.RetryRaw + public ObjectListingIterator createObjectListingIterator( + final Path listPath, + final S3ListRequest request) throws IOException { + return new ObjectListingIterator(listPath, request); + } + /** * Create a located status iterator over a file status iterator. * @param statusIterator an iterator over the remote status entries @@ -194,8 +209,12 @@ public RemoteIterator getListFilesAssumingDir( String key = maybeAddTrailingSlash(pathToKey(path)); String delimiter = recursive ? null : "/"; - LOG.debug("Requesting all entries under {} with delimiter '{}'", - key, delimiter); + if (recursive) { + LOG.debug("Recursive list of all entries under {}", key); + } else { + LOG.debug("Requesting all entries under {} with delimiter '{}'", + key, delimiter); + } final RemoteIterator cachedFilesIterator; final Set tombstones; boolean allowAuthoritative = listingOperationCallbacks diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 2cd23255c4b26..ac9904a867e21 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -67,7 +67,7 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3ObjectSummary; + import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams; import com.amazonaws.services.s3.model.SSECustomerKey; import com.amazonaws.services.s3.model.UploadPartRequest; @@ -104,6 +104,8 @@ import org.apache.hadoop.fs.s3a.impl.ContextAccessors; import org.apache.hadoop.fs.s3a.impl.CopyOutcome; import org.apache.hadoop.fs.s3a.impl.DeleteOperation; +import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; +import org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl; import org.apache.hadoop.fs.s3a.impl.InternalConstants; import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks; import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport; @@ -116,6 +118,8 @@ import org.apache.hadoop.fs.s3a.impl.statistics.S3AMultipartUploaderStatisticsImpl; import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState; import org.apache.hadoop.fs.s3a.select.InternalSelectConstants; +import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations; +import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.DelegationTokenIssuer; @@ -295,6 +299,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private final ListingOperationCallbacks listingOperationCallbacks = new ListingOperationCallbacksImpl(); + /** + * Directory policy. + */ + private DirectoryPolicy directoryPolicy; + + /** + * Context accessors for re-use. + */ + private final ContextAccessors contextAccessors = new ContextAccessorsImpl(); /** Add any deprecated keys. */ @SuppressWarnings("deprecation") @@ -452,6 +465,10 @@ public void initialize(URI name, Configuration originalConf) DEFAULT_S3GUARD_DISABLED_WARN_LEVEL); S3Guard.logS3GuardDisabled(LOG, warnLevel, bucket); } + // directory policy, which may look at authoritative paths + directoryPolicy = DirectoryPolicyImpl.getDirectoryPolicy(conf, + this::allowAuthoritative); + LOG.debug("Directory marker retention policy is {}", directoryPolicy); initMultipartUploads(conf); @@ -1285,7 +1302,7 @@ public WriteOperationHelper getWriteOperationHelper() { * is not a directory. */ @Override - public FSDataOutputStream createNonRecursive(Path path, + public FSDataOutputStream createNonRecursive(Path p, FsPermission permission, EnumSet flags, int bufferSize, @@ -1293,10 +1310,22 @@ public FSDataOutputStream createNonRecursive(Path path, long blockSize, Progressable progress) throws IOException { entryPoint(INVOCATION_CREATE_NON_RECURSIVE); + final Path path = makeQualified(p); Path parent = path.getParent(); - if (parent != null) { - // expect this to raise an exception if there is no parent - if (!getFileStatus(parent).isDirectory()) { + // expect this to raise an exception if there is no parent dir + if (parent != null && !parent.isRoot()) { + S3AFileStatus status; + try { + // optimize for the directory existing: Call list first + status = innerGetFileStatus(parent, false, + StatusProbeEnum.DIRECTORIES); + } catch (FileNotFoundException e) { + // no dir, fall back to looking for a file + // (failure condition if true) + status = innerGetFileStatus(parent, false, + StatusProbeEnum.HEAD_ONLY); + } + if (!status.isDirectory()) { throw new FileAlreadyExistsException("Not a directory: " + parent); } } @@ -1431,10 +1460,13 @@ private Pair initiateRename( LOG.debug("rename: destination path {} not found", dst); // Parent must exist Path parent = dst.getParent(); - if (!pathToKey(parent).isEmpty()) { + if (!pathToKey(parent).isEmpty() + && !parent.equals(src.getParent())) { try { - S3AFileStatus dstParentStatus = innerGetFileStatus(dst.getParent(), - false, StatusProbeEnum.ALL); + // only look against S3 for directories; saves + // a HEAD request on all normal operations. + S3AFileStatus dstParentStatus = innerGetFileStatus(parent, + false, StatusProbeEnum.DIRECTORIES); if (!dstParentStatus.isDirectory()) { throw new RenameFailedException(src, dst, "destination parent is not a directory"); @@ -1535,7 +1567,7 @@ public void deleteObjectAtPath(final Path path, final boolean isFile, final BulkOperationState operationState) throws IOException { - once("delete", key, () -> + once("delete", path.toString(), () -> S3AFileSystem.this.deleteObjectAtPath(path, key, isFile, operationState)); } @@ -1585,7 +1617,9 @@ public void finishRename(final Path sourceRenamed, final Path destCreated) Path destParent = destCreated.getParent(); if (!sourceRenamed.getParent().equals(destParent)) { LOG.debug("source & dest parents are different; fix up dir markers"); - deleteUnnecessaryFakeDirectories(destParent); + if (!keepDirectoryMarkers(destParent)) { + deleteUnnecessaryFakeDirectories(destParent, null); + } maybeCreateFakeParentDirectory(sourceRenamed); } } @@ -1940,6 +1974,7 @@ protected ObjectMetadata getObjectMetadata(String key, protected S3ListResult listObjects(S3ListRequest request) throws IOException { incrementReadOperations(); incrementStatistic(OBJECT_LIST_REQUESTS); + LOG.debug("LIST {}", request); validateListArguments(request); try(DurationInfo ignored = new DurationInfo(LOG, false, "LIST")) { @@ -2381,6 +2416,14 @@ private DeleteObjectsResult removeKeysS3( boolean quiet) throws MultiObjectDeleteException, AmazonClientException, IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Initiating delete operation for {} objects", + keysToDelete.size()); + for (DeleteObjectsRequest.KeyVersion key : keysToDelete) { + LOG.debug(" {} {}", key.getKey(), + key.getVersion() != null ? key.getVersion() : ""); + } + } DeleteObjectsResult result = null; if (keysToDelete.isEmpty()) { // exit fast if there are no keys to delete @@ -2490,7 +2533,8 @@ DeleteObjectsResult removeKeys( final boolean quiet) throws MultiObjectDeleteException, AmazonClientException, IOException { undeletedObjectsOnFailure.clear(); - try (DurationInfo ignored = new DurationInfo(LOG, false, "Deleting")) { + try (DurationInfo ignored = new DurationInfo(LOG, false, + "Deleting %d keys", keysToDelete.size())) { return removeKeysS3(keysToDelete, deleteFakeDir, quiet); } catch (MultiObjectDeleteException ex) { LOG.debug("Partial delete failure"); @@ -2573,7 +2617,7 @@ private void createFakeDirectoryIfNecessary(Path f) // we only make the LIST call; the codepaths to get here should not // be reached if there is an empty dir marker -and if they do, it // is mostly harmless to create a new one. - if (!key.isEmpty() && !s3Exists(f, EnumSet.of(StatusProbeEnum.List))) { + if (!key.isEmpty() && !s3Exists(f, StatusProbeEnum.DIRECTORIES)) { LOG.debug("Creating new fake directory at {}", f); createFakeDirectory(key); } @@ -2589,7 +2633,7 @@ private void createFakeDirectoryIfNecessary(Path f) void maybeCreateFakeParentDirectory(Path path) throws IOException, AmazonClientException { Path parent = path.getParent(); - if (parent != null) { + if (parent != null && !parent.isRoot()) { createFakeDirectoryIfNecessary(parent); } } @@ -2618,7 +2662,7 @@ public FileStatus[] listStatus(Path f) throws FileNotFoundException, * @throws IOException due to an IO problem. * @throws AmazonClientException on failures inside the AWS SDK */ - public FileStatus[] innerListStatus(Path f) throws FileNotFoundException, + private S3AFileStatus[] innerListStatus(Path f) throws FileNotFoundException, IOException, AmazonClientException { Path path = qualify(f); String key = pathToKey(path); @@ -2626,7 +2670,8 @@ public FileStatus[] innerListStatus(Path f) throws FileNotFoundException, entryPoint(INVOCATION_LIST_STATUS); List result; - final FileStatus fileStatus = getFileStatus(path); + final S3AFileStatus fileStatus = innerGetFileStatus(path, false, + StatusProbeEnum.ALL); if (fileStatus.isDirectory()) { if (!key.isEmpty()) { @@ -2658,7 +2703,7 @@ public FileStatus[] innerListStatus(Path f) throws FileNotFoundException, allowAuthoritative, ttlTimeProvider); } else { LOG.debug("Adding: rd (not a dir): {}", path); - FileStatus[] stats = new FileStatus[1]; + S3AFileStatus[] stats = new S3AFileStatus[1]; stats[0]= fileStatus; return stats; } @@ -2769,9 +2814,10 @@ public UserGroupInformation getOwner() { public boolean mkdirs(Path path, FsPermission permission) throws IOException, FileAlreadyExistsException { try { + entryPoint(INVOCATION_MKDIRS); return innerMkdirs(path, permission); } catch (AmazonClientException e) { - throw translateException("innerMkdirs", path, e); + throw translateException("mkdirs", path, e); } } @@ -2791,11 +2837,15 @@ private boolean innerMkdirs(Path p, FsPermission permission) throws IOException, FileAlreadyExistsException, AmazonClientException { Path f = qualify(p); LOG.debug("Making directory: {}", f); - entryPoint(INVOCATION_MKDIRS); + if (p.isRoot()) { + // fast exit for root. + return true; + } FileStatus fileStatus; try { - fileStatus = getFileStatus(f); + fileStatus = innerGetFileStatus(f, false, + StatusProbeEnum.ALL); if (fileStatus.isDirectory()) { return true; @@ -2805,7 +2855,7 @@ private boolean innerMkdirs(Path p, FsPermission permission) } catch (FileNotFoundException e) { // Walk path to root, ensuring closest ancestor is a directory, not file Path fPart = f.getParent(); - while (fPart != null) { + while (fPart != null && !fPart.isRoot()) { try { fileStatus = getFileStatus(fPart); if (fileStatus.isDirectory()) { @@ -2866,7 +2916,8 @@ S3AFileStatus innerGetFileStatus(final Path f, final Set probes) throws IOException { final Path path = qualify(f); String key = pathToKey(path); - LOG.debug("Getting path status for {} ({})", path, key); + LOG.debug("Getting path status for {} ({}); needEmptyDirectory={}", + path, key, needEmptyDirectoryFlag); boolean allowAuthoritative = allowAuthoritative(path); // Check MetadataStore, if any. @@ -2877,9 +2928,10 @@ S3AFileStatus innerGetFileStatus(final Path f, } Set tombstones = Collections.emptySet(); if (pm != null) { + S3AFileStatus msStatus = pm.getFileStatus(); if (pm.isDeleted()) { OffsetDateTime deletedAt = OffsetDateTime.ofInstant( - Instant.ofEpochMilli(pm.getFileStatus().getModificationTime()), + Instant.ofEpochMilli(msStatus.getModificationTime()), ZoneOffset.UTC); throw new FileNotFoundException("Path " + path + " is recorded as " + "deleted by S3Guard at " + deletedAt); @@ -2890,72 +2942,114 @@ S3AFileStatus innerGetFileStatus(final Path f, // Skip going to s3 if the file checked is a directory. Because if the // dest is also a directory, there's no difference. - if (!pm.getFileStatus().isDirectory() && + if (!msStatus.isDirectory() && !allowAuthoritative && probes.contains(StatusProbeEnum.Head)) { // a file has been found in a non-auth path and the caller has not said // they only care about directories LOG.debug("Metadata for {} found in the non-auth metastore.", path); - final long msModTime = pm.getFileStatus().getModificationTime(); - - S3AFileStatus s3AFileStatus; - try { - s3AFileStatus = s3GetFileStatus(path, key, probes, tombstones); - } catch (FileNotFoundException fne) { - s3AFileStatus = null; - } - if (s3AFileStatus == null) { - LOG.warn("Failed to find file {}. Either it is not yet visible, or " - + "it has been deleted.", path); - } else { - final long s3ModTime = s3AFileStatus.getModificationTime(); - - if(s3ModTime > msModTime) { - LOG.debug("S3Guard metadata for {} is outdated;" - + " s3modtime={}; msModTime={} updating metastore", - path, s3ModTime, msModTime); - return S3Guard.putAndReturn(metadataStore, s3AFileStatus, - ttlTimeProvider); + // If the timestamp of the pm is close to "now", we don't need to + // bother with a check of S3. that means: + // one of : status modtime is close to now, + // or pm.getLastUpdated() == now + + // get the time in which a status modtime is considered valid + // in a non-auth metastore + long validTime = + ttlTimeProvider.getNow() - ttlTimeProvider.getMetadataTtl(); + final long msModTime = msStatus.getModificationTime(); + + if (msModTime < validTime) { + LOG.debug("Metastore entry of {} is out of date, probing S3", path); + try { + S3AFileStatus s3AFileStatus = s3GetFileStatus(path, + key, + probes, + tombstones, + needEmptyDirectoryFlag); + // if the new status is more current than that in the metastore, + // it means S3 has changed and the store needs updating + final long s3ModTime = s3AFileStatus.getModificationTime(); + + if (s3ModTime > msModTime) { + // there's new data in S3 + LOG.debug("S3Guard metadata for {} is outdated;" + + " s3modtime={}; msModTime={} updating metastore", + path, s3ModTime, msModTime); + // add to S3Guard + S3Guard.putAndReturn(metadataStore, s3AFileStatus, + ttlTimeProvider); + } else { + // the modtime of the data is the same as/older than the s3guard + // value either an old object has been found, or the existing one + // was retrieved in both cases -refresh the S3Guard entry so the + // record's TTL is updated. + S3Guard.refreshEntry(metadataStore, pm, s3AFileStatus, + ttlTimeProvider); + } + // return the value + // note that the checks for empty dir status below can be skipped + // because the call to s3GetFileStatus include the checks there + return s3AFileStatus; + } catch (FileNotFoundException fne) { + // the attempt to refresh the record failed because there was + // no entry. Either it is a new file not visible, or it + // has been deleted (and therefore S3Guard is out of sync with S3) + LOG.warn("Failed to find file {}. Either it is not yet visible, or " + + "it has been deleted.", path); } } } - S3AFileStatus msStatus = pm.getFileStatus(); if (needEmptyDirectoryFlag && msStatus.isDirectory()) { + // the caller needs to know if a directory is empty, + // and that this is a directory. if (pm.isEmptyDirectory() != Tristate.UNKNOWN) { // We have a definitive true / false from MetadataStore, we are done. return msStatus; } else { + // execute a S3Guard listChildren command to list tombstones under the + // path. + // This list will be used in the forthcoming s3GetFileStatus call. DirListingMetadata children = S3Guard.listChildrenWithTtl(metadataStore, path, ttlTimeProvider, allowAuthoritative); if (children != null) { tombstones = children.listTombstones(); } - LOG.debug("MetadataStore doesn't know if dir is empty, using S3."); + LOG.debug("MetadataStore doesn't know if {} is empty, using S3.", + path); } } else { // Either this is not a directory, or we don't care if it is empty return msStatus; } - // If the metadata store has no children for it and it's not listed in - // S3 yet, we'll assume the empty directory is true; - S3AFileStatus s3FileStatus; + // now issue the S3 getFileStatus call. try { - s3FileStatus = s3GetFileStatus(path, key, probes, tombstones); + S3AFileStatus s3FileStatus = s3GetFileStatus(path, + key, + probes, + tombstones, + true); + // entry was found, so save in S3Guard and return the final value. + return S3Guard.putAndReturn(metadataStore, s3FileStatus, + ttlTimeProvider); } catch (FileNotFoundException e) { + // If the metadata store has no children for it and it's not listed in + // S3 yet, we'll conclude that it is an empty directory return S3AFileStatus.fromFileStatus(msStatus, Tristate.TRUE, null, null); } - // entry was found, save in S3Guard - return S3Guard.putAndReturn(metadataStore, s3FileStatus, - ttlTimeProvider); } else { // there was no entry in S3Guard // retrieve the data and update the metadata store in the process. return S3Guard.putAndReturn(metadataStore, - s3GetFileStatus(path, key, probes, tombstones), + s3GetFileStatus(path, + key, + probes, + tombstones, + needEmptyDirectoryFlag), ttlTimeProvider); } } @@ -3010,6 +3104,8 @@ S3AFileStatus innerGetFileStatus(final Path f, * @param key Key string for the path * @param probes probes to make * @param tombstones tombstones to filter + * @param needEmptyDirectoryFlag if true, implementation will calculate + * a TRUE or FALSE value for {@link S3AFileStatus#isEmptyDirectory()} * @return Status * @throws FileNotFoundException the supplied probes failed. * @throws IOException on other problems. @@ -3019,88 +3115,88 @@ S3AFileStatus innerGetFileStatus(final Path f, S3AFileStatus s3GetFileStatus(final Path path, final String key, final Set probes, - @Nullable Set tombstones) throws IOException { - if (!key.isEmpty()) { - if (probes.contains(StatusProbeEnum.Head) && !key.endsWith("/")) { - try { - // look for the simple file - ObjectMetadata meta = getObjectMetadata(key); - LOG.debug("Found exact file: normal file {}", key); - return new S3AFileStatus(meta.getContentLength(), - dateToLong(meta.getLastModified()), - path, - getDefaultBlockSize(path), - username, - meta.getETag(), - meta.getVersionId()); - } catch (AmazonServiceException e) { - // if the response is a 404 error, it just means that there is - // no file at that path...the remaining checks will be needed. - if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) { - throw translateException("getFileStatus", path, e); - } - } catch (AmazonClientException e) { + @Nullable final Set tombstones, + final boolean needEmptyDirectoryFlag) throws IOException { + LOG.debug("S3GetFileStatus {}", path); + // either you aren't looking for the directory flag, or you are, + // and if you are, the probe list must contain list. + Preconditions.checkArgument(!needEmptyDirectoryFlag + || probes.contains(StatusProbeEnum.List), + "s3GetFileStatus(%s) wants to know if a directory is empty but" + + " does not request a list probe", path); + + if (!key.isEmpty() && !key.endsWith("/") + && probes.contains(StatusProbeEnum.Head)) { + try { + // look for the simple file + ObjectMetadata meta = getObjectMetadata(key); + LOG.debug("Found exact file: normal file {}", key); + return new S3AFileStatus(meta.getContentLength(), + dateToLong(meta.getLastModified()), + path, + getDefaultBlockSize(path), + username, + meta.getETag(), + meta.getVersionId()); + } catch (AmazonServiceException e) { + // if the response is a 404 error, it just means that there is + // no file at that path...the remaining checks will be needed. + if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) { throw translateException("getFileStatus", path, e); } - } - - // Either a normal file was not found or the probe was skipped. - // because the key ended in "/" or it was not in the set of probes. - // Look for the dir marker - if (probes.contains(StatusProbeEnum.DirMarker)) { - String newKey = maybeAddTrailingSlash(key); - try { - ObjectMetadata meta = getObjectMetadata(newKey); - - if (objectRepresentsDirectory(newKey, meta.getContentLength())) { - LOG.debug("Found file (with /): fake directory"); - return new S3AFileStatus(Tristate.TRUE, path, username); - } else { - LOG.warn("Found file (with /): real file? should not happen: {}", - key); - - return new S3AFileStatus(meta.getContentLength(), - dateToLong(meta.getLastModified()), - path, - getDefaultBlockSize(path), - username, - meta.getETag(), - meta.getVersionId()); - } - } catch (AmazonServiceException e) { - if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) { - throw translateException("getFileStatus", newKey, e); - } - } catch (AmazonClientException e) { - throw translateException("getFileStatus", newKey, e); - } + } catch (AmazonClientException e) { + throw translateException("getFileStatus", path, e); } } // execute the list if (probes.contains(StatusProbeEnum.List)) { try { + // this will find a marker dir / as well as an entry. + // When making a simple "is this a dir check" all is good. + // but when looking for an empty dir, we need to verify there are no + // children, so ask for two entries, so as to find + // a child String dirKey = maybeAddTrailingSlash(key); - S3ListRequest request = createListObjectsRequest(dirKey, "/", 1); + // list size is dir marker + at least one non-tombstone entry + // there's a corner case: more tombstones than you have in a + // single page list. We assume that if you have been deleting + // that many files, then the AWS listing will have purged some + // by the time of listing so that the response includes some + // which have not. + + int listSize; + if (tombstones == null) { + // no tombstones so look for a marker and at least one child. + listSize = 2; + } else { + // build a listing > tombstones. If the caller has many thousands + // of tombstones this won't work properly, which is why pruning + // of expired tombstones matters. + listSize = Math.min(2 + tombstones.size(), Math.max(2, maxKeys)); + } + S3ListRequest request = createListObjectsRequest(dirKey, "/", + listSize); + // execute the request + S3ListResult listResult = listObjects(request); - S3ListResult objects = listObjects(request); - Collection prefixes = objects.getCommonPrefixes(); - Collection summaries = objects.getObjectSummaries(); - if (!isEmptyOfKeys(prefixes, tombstones) || - !isEmptyOfObjects(summaries, tombstones)) { + if (listResult.hasPrefixesOrObjects(contextAccessors, tombstones)) { if (LOG.isDebugEnabled()) { - LOG.debug("Found path as directory (with /): {}/{}", - prefixes.size(), summaries.size()); - - for (S3ObjectSummary summary : summaries) { - LOG.debug("Summary: {} {}", summary.getKey(), summary.getSize()); - } - for (String prefix : prefixes) { - LOG.debug("Prefix: {}", prefix); - } + LOG.debug("Found path as directory (with /)"); + listResult.logAtDebug(LOG); } - + // At least one entry has been found. + // If looking for an empty directory, the marker must exist but no + // children. + // So the listing must contain the marker entry only. + if (needEmptyDirectoryFlag + && listResult.representsEmptyDirectory( + contextAccessors, dirKey, tombstones)) { + return new S3AFileStatus(Tristate.TRUE, path, username); + } + // either an empty directory is not needed, or the + // listing does not meet the requirements. return new S3AFileStatus(Tristate.FALSE, path, username); } else if (key.isEmpty()) { LOG.debug("Found root directory"); @@ -3119,48 +3215,6 @@ S3AFileStatus s3GetFileStatus(final Path path, throw new FileNotFoundException("No such file or directory: " + path); } - /** - * Helper function to determine if a collection of paths is empty - * after accounting for tombstone markers (if provided). - * @param keys Collection of path (prefixes / directories or keys). - * @param tombstones Set of tombstone markers, or null if not applicable. - * @return false if summaries contains objects not accounted for by - * tombstones. - */ - private boolean isEmptyOfKeys(Collection keys, Set - tombstones) { - if (tombstones == null) { - return keys.isEmpty(); - } - for (String key : keys) { - Path qualified = keyToQualifiedPath(key); - if (!tombstones.contains(qualified)) { - return false; - } - } - return true; - } - - /** - * Helper function to determine if a collection of object summaries is empty - * after accounting for tombstone markers (if provided). - * @param summaries Collection of objects as returned by listObjects. - * @param tombstones Set of tombstone markers, or null if not applicable. - * @return false if summaries contains objects not accounted for by - * tombstones. - */ - private boolean isEmptyOfObjects(Collection summaries, - Set tombstones) { - if (tombstones == null) { - return summaries.isEmpty(); - } - Collection stringCollection = new ArrayList<>(summaries.size()); - for (S3ObjectSummary summary : summaries) { - stringCollection.add(summary.getKey()); - } - return isEmptyOfKeys(stringCollection, tombstones); - } - /** * Raw version of {@link FileSystem#exists(Path)} which uses S3 only: * S3Guard MetadataStore, if any, will be skipped. @@ -3175,7 +3229,7 @@ private boolean s3Exists(final Path path, final Set probes) throws IOException { String key = pathToKey(path); try { - s3GetFileStatus(path, key, probes, null); + s3GetFileStatus(path, key, probes, null, false); return true; } catch (FileNotFoundException e) { return false; @@ -3578,6 +3632,7 @@ private CopyResult copyFile(String srcKey, String dstKey, long size, copyObjectRequest.setNewObjectMetadata(dstom); Optional.ofNullable(srcom.getStorageClass()) .ifPresent(copyObjectRequest::setStorageClass); + incrementStatistic(OBJECT_COPY_REQUESTS); Copy copy = transfers.copy(copyObjectRequest); copy.addProgressListener(progressListener); CopyOutcome copyOutcome = CopyOutcome.waitForCopy(copy); @@ -3711,16 +3766,21 @@ private Optional generateSSECustomerKey() { /** * Perform post-write actions. - * Calls {@link #deleteUnnecessaryFakeDirectories(Path)} and then - * updates any metastore. + *

* This operation MUST be called after any PUT/multipart PUT completes * successfully. - * - * The operations actions include + *

+ * The actions include: *
    - *
  1. Calling {@link #deleteUnnecessaryFakeDirectories(Path)}
    2. - *
  2. Updating any metadata store with details on the newly created - * object.
    3. + *
  3. + * Calling + * {@link #deleteUnnecessaryFakeDirectories(Path, BulkOperationState)} + * if directory markers are not being retained. + *
    4. + *
  4. + * Updating any metadata store with details on the newly created + * object. + *
    5. *
* @param key key written to * @param length total length of file written @@ -3743,12 +3803,19 @@ void finishedWrite(String key, long length, String eTag, String versionId, Preconditions.checkArgument(length >= 0, "content length is negative"); final boolean isDir = objectRepresentsDirectory(key, length); // kick off an async delete - final CompletableFuture deletion = submit( - unboundedThreadPool, - () -> { - deleteUnnecessaryFakeDirectories(p.getParent()); - return null; - }); + CompletableFuture deletion; + if (!keepDirectoryMarkers(p)) { + deletion = submit( + unboundedThreadPool, + () -> { + deleteUnnecessaryFakeDirectories( + p.getParent(), + operationState); + return null; + }); + } else { + deletion = null; + } // this is only set if there is a metastore to update and the // operationState parameter passed in was null. BulkOperationState stateToClose = null; @@ -3807,13 +3874,26 @@ void finishedWrite(String key, long length, String eTag, String versionId, } } + /** + * Should we keep directory markers under the path being created + * by mkdir/file creation/rename? + * @param path path to probe + * @return true if the markers MAY be retained, + * false if they MUST be deleted + */ + private boolean keepDirectoryMarkers(Path path) { + return directoryPolicy.keepDirectoryMarkers(path); + } + /** * Delete mock parent directories which are no longer needed. * Retry policy: retrying; exceptions swallowed. * @param path path + * @param operationState (nullable) operational state for a bulk update */ @Retries.RetryExceptionsSwallowed - private void deleteUnnecessaryFakeDirectories(Path path) { + private void deleteUnnecessaryFakeDirectories(Path path, + final BulkOperationState operationState) { List keysToRemove = new ArrayList<>(); while (!path.isRoot()) { String key = pathToKey(path); @@ -3823,7 +3903,7 @@ private void deleteUnnecessaryFakeDirectories(Path path) { path = path.getParent(); } try { - removeKeys(keysToRemove, true, null); + removeKeys(keysToRemove, true, operationState); } catch(AmazonClientException | IOException e) { instrumentation.errorIgnored(); if (LOG.isDebugEnabled()) { @@ -3952,6 +4032,14 @@ public long getDefaultBlockSize() { return getConf().getLongBytes(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE); } + /** + * Get the directory marker policy of this filesystem. + * @return the marker policy. + */ + public DirectoryPolicy getDirectoryMarkerPolicy() { + return directoryPolicy; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder( @@ -3990,6 +4078,7 @@ public String toString() { sb.append(", credentials=").append(credentials); sb.append(", delegation tokens=") .append(delegationTokens.map(Objects::toString).orElse("disabled")); + sb.append(", ").append(directoryPolicy); sb.append(", statistics {") .append(statistics) .append("}"); @@ -4086,25 +4175,41 @@ public boolean exists(Path f) throws IOException { } /** - * Override superclass so as to add statistic collection. + * Optimized probe for a path referencing a dir. + * Even though it is optimized to a single HEAD, applications + * should not over-use this method...it is all too common. * {@inheritDoc} */ @Override @SuppressWarnings("deprecation") public boolean isDirectory(Path f) throws IOException { entryPoint(INVOCATION_IS_DIRECTORY); - return super.isDirectory(f); + try { + return innerGetFileStatus(f, false, StatusProbeEnum.DIRECTORIES) + .isDirectory(); + } catch (FileNotFoundException e) { + // not found or it is a file. + return false; + } } /** - * Override superclass so as to add statistic collection. + * Optimized probe for a path referencing a file. + * Even though it is optimized to a single HEAD, applications + * should not over-use this method...it is all too common. * {@inheritDoc} */ @Override @SuppressWarnings("deprecation") public boolean isFile(Path f) throws IOException { entryPoint(INVOCATION_IS_FILE); - return super.isFile(f); + try { + return innerGetFileStatus(f, false, StatusProbeEnum.HEAD_ONLY) + .isFile(); + } catch (FileNotFoundException e) { + // not found or it is a dir. + return false; + } } /** @@ -4511,7 +4616,8 @@ public S3AInstrumentation.CommitterStatistics newCommitterStatistics() { public boolean hasPathCapability(final Path path, final String capability) throws IOException { final Path p = makeQualified(path); - switch (validatePathCapabilityArgs(p, capability)) { + String cap = validatePathCapabilityArgs(p, capability); + switch (cap) { case CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER: case CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER_OLD: @@ -4530,8 +4636,24 @@ public boolean hasPathCapability(final Path path, final String capability) case CommonPathCapabilities.FS_MULTIPART_UPLOADER: return true; + // this client is safe to use with buckets + // containing directory markers anywhere in + // the hierarchy + case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE: + return true; + + /* + * Marker policy capabilities are handed off. + */ + case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP: + case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE: + case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE: + case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP: + case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE: + return getDirectoryMarkerPolicy().hasPathCapability(path, cap); + default: - return super.hasPathCapability(p, capability); + return super.hasPathCapability(p, cap); } } @@ -4546,7 +4668,7 @@ public boolean hasPathCapability(final Path path, final String capability) @Override public boolean hasCapability(String capability) { try { - return hasPathCapability(workingDir, capability); + return hasPathCapability(new Path("/"), capability); } catch (IOException ex) { // should never happen, so log and downgrade. LOG.debug("Ignoring exception on hasCapability({}})", capability, ex); @@ -4800,6 +4922,15 @@ public StoreContext createStoreContext() { .build(); } + /** + * Create a marker tools operations binding for this store. + * @return callbacks for operations. + */ + @InterfaceAudience.Private + public MarkerToolOperations createMarkerToolOperations() { + return new MarkerToolOperationsImpl(operationCallbacks); + } + /** * The implementation of context accessors. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java index 1a0d2c3378ca6..d51211516f251 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java @@ -24,11 +24,18 @@ /** * API version-independent container for S3 List requests. */ -public class S3ListRequest { - private ListObjectsRequest v1Request; - private ListObjectsV2Request v2Request; +public final class S3ListRequest { - protected S3ListRequest(ListObjectsRequest v1, ListObjectsV2Request v2) { + /** + * Format for the toString() method: {@value}. + */ + private static final String DESCRIPTION + = "List %s:/%s delimiter=%s keys=%d requester pays=%s"; + + private final ListObjectsRequest v1Request; + private final ListObjectsV2Request v2Request; + + private S3ListRequest(ListObjectsRequest v1, ListObjectsV2Request v2) { v1Request = v1; v2Request = v2; } @@ -70,11 +77,15 @@ public ListObjectsV2Request getV2() { @Override public String toString() { if (isV1()) { - return String.format("List %s:/%s", - v1Request.getBucketName(), v1Request.getPrefix()); + return String.format(DESCRIPTION, + v1Request.getBucketName(), v1Request.getPrefix(), + v1Request.getDelimiter(), v1Request.getMaxKeys(), + v1Request.isRequesterPays()); } else { - return String.format("List %s:/%s", - v2Request.getBucketName(), v2Request.getPrefix()); + return String.format(DESCRIPTION, + v2Request.getBucketName(), v2Request.getPrefix(), + v2Request.getDelimiter(), v2Request.getMaxKeys(), + v2Request.isRequesterPays()); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java index e8aff329070ef..69794c04db53c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java @@ -18,11 +18,18 @@ package org.apache.hadoop.fs.s3a; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.slf4j.Logger; -import java.util.List; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.impl.ContextAccessors; /** * API version-independent container for S3 List responses. @@ -92,6 +99,110 @@ public List getCommonPrefixes() { } else { return v2Result.getCommonPrefixes(); } + } + + /** + * Is the list of object summaries empty + * after accounting for tombstone markers (if provided)? + * @param accessors callback for key to path mapping. + * @param tombstones Set of tombstone markers, or null if not applicable. + * @return false if summaries contains objects not accounted for by + * tombstones. + */ + public boolean isEmptyOfObjects( + final ContextAccessors accessors, + final Set tombstones) { + if (tombstones == null) { + return getObjectSummaries().isEmpty(); + } + return isEmptyOfKeys(accessors, + objectSummaryKeys(), + tombstones); + } + + /** + * Get the list of keys in the object summary. + * @return a possibly empty list + */ + private List objectSummaryKeys() { + return getObjectSummaries().stream() + .map(S3ObjectSummary::getKey) + .collect(Collectors.toList()); + } + + /** + * Does this listing have prefixes or objects after entries with + * tombstones have been stripped? + * @param accessors callback for key to path mapping. + * @param tombstones Set of tombstone markers, or null if not applicable. + * @return true if the reconciled list is non-empty + */ + public boolean hasPrefixesOrObjects( + final ContextAccessors accessors, + final Set tombstones) { + + return !isEmptyOfKeys(accessors, getCommonPrefixes(), tombstones) + || !isEmptyOfObjects(accessors, tombstones); + } + + /** + * Helper function to determine if a collection of keys is empty + * after accounting for tombstone markers (if provided). + * @param accessors callback for key to path mapping. + * @param keys Collection of path (prefixes / directories or keys). + * @param tombstones Set of tombstone markers, or null if not applicable. + * @return true if the list is considered empty. + */ + public boolean isEmptyOfKeys( + final ContextAccessors accessors, + final Collection keys, + final Set tombstones) { + if (tombstones == null) { + return keys.isEmpty(); + } + for (String key : keys) { + Path qualified = accessors.keyToPath(key); + if (!tombstones.contains(qualified)) { + return false; + } + } + return true; + } + /** + * Does this listing represent an empty directory? + * @param contextAccessors callback for key to path mapping. + * @param dirKey directory key + * @param tombstones Set of tombstone markers, or null if not applicable. + * @return true if the list is considered empty. + */ + public boolean representsEmptyDirectory( + final ContextAccessors contextAccessors, + final String dirKey, + final Set tombstones) { + // If looking for an empty directory, the marker must exist but + // no children. + // So the listing must contain the marker entry only as an object, + // and prefixes is null + List keys = objectSummaryKeys(); + return keys.size() == 1 && keys.contains(dirKey) + && getCommonPrefixes().isEmpty(); + } + + /** + * Dmp the result at debug level. + * @param log log to use + */ + public void logAtDebug(Logger log) { + Collection prefixes = getCommonPrefixes(); + Collection summaries = getObjectSummaries(); + log.debug("Prefix count = {}; object count={}", + prefixes.size(), summaries.size()); + for (S3ObjectSummary summary : summaries) { + log.debug("Summary: {} {}", summary.getKey(), summary.getSize()); + } + for (String prefix : prefixes) { + log.debug("Prefix: {}", prefix); + } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java new file mode 100644 index 0000000000000..ca04fed65a539 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus; + +/** + * Tracks directory markers which have been reported in object listings. + * This is needed for auditing and cleanup, including during rename + * operations. + *

+ * Designed to be used while scanning through the results of listObject + * calls, where are we assume the results come in alphanumeric sort order + * and parent entries before children. + *

+ * This lets as assume that we can identify all leaf markers as those + * markers which were added to set of leaf markers and not subsequently + * removed as a child entries were discovered. + *

+ * To avoid scanning datastructures excessively, the path of the parent + * directory of the last file added is cached. This allows for a + * quick bailout when many children of the same directory are + * returned in a listing. + *

+ * Consult the directory_markers document for details on this feature, + * including terminology. + */ +public class DirMarkerTracker { + + private static final Logger LOG = + LoggerFactory.getLogger(DirMarkerTracker.class); + + /** + * all leaf markers. + */ + private final Map leafMarkers + = new TreeMap<>(); + + /** + * all surplus markers. + */ + private final Map surplusMarkers + = new TreeMap<>(); + + /** + * Base path of the tracking operation. + */ + private final Path basePath; + + /** + * Should surplus markers be recorded in + * the {@link #surplusMarkers} map? + */ + private final boolean recordSurplusMarkers; + + /** + * last parent directory checked. + */ + private Path lastDirChecked; + + /** + * Count of scans; used for test assertions. + */ + private int scanCount; + + /** + * How many files were found. + */ + private int filesFound; + + /** + * How many markers were found. + */ + private int markersFound; + + /** + * How many objects of any kind were found? + */ + private int objectsFound; + + /** + * Construct. + *

+ * The base path is currently only used for information rather than + * validating paths supplied in other methods. + * @param basePath base path of track + * @param recordSurplusMarkers save surplus markers to a map? + */ + public DirMarkerTracker(final Path basePath, + boolean recordSurplusMarkers) { + this.basePath = basePath; + this.recordSurplusMarkers = recordSurplusMarkers; + } + + /** + * Get the base path of the tracker. + * @return the path + */ + public Path getBasePath() { + return basePath; + } + + /** + * A marker has been found; this may or may not be a leaf. + *

+ * Trigger a move of all markers above it into the surplus map. + * @param path marker path + * @param key object key + * @param source listing source + * @return the surplus markers found. + */ + public List markerFound(Path path, + final String key, + final S3ALocatedFileStatus source) { + markersFound++; + leafMarkers.put(path, new Marker(path, key, source)); + return pathFound(path, key, source); + } + + /** + * A file has been found. Trigger a move of all + * markers above it into the surplus map. + * @param path marker path + * @param key object key + * @param source listing source + * @return the surplus markers found. + */ + public List fileFound(Path path, + final String key, + final S3ALocatedFileStatus source) { + filesFound++; + return pathFound(path, key, source); + } + + /** + * A path has been found. + *

+ * Declare all markers above it as surplus + * @param path marker path + * @param key object key + * @param source listing source + * @return the surplus markers found. + */ + private List pathFound(Path path, + final String key, + final S3ALocatedFileStatus source) { + objectsFound++; + List removed = new ArrayList<>(); + + // all parent entries are superfluous + final Path parent = path.getParent(); + if (parent == null || parent.equals(lastDirChecked)) { + // short cut exit + return removed; + } + removeParentMarkers(parent, removed); + lastDirChecked = parent; + return removed; + } + + /** + * Remove all markers from the path and its parents from the + * {@link #leafMarkers} map. + *

+ * if {@link #recordSurplusMarkers} is true, the marker is + * moved to the surplus map. Not doing this is simply an + * optimisation designed to reduce risk of excess memory consumption + * when renaming (hypothetically) large directory trees. + * @param path path to start at + * @param removed list of markers removed; is built up during the + * recursive operation. + */ + private void removeParentMarkers(final Path path, + List removed) { + if (path == null || path.isRoot()) { + return; + } + scanCount++; + removeParentMarkers(path.getParent(), removed); + final Marker value = leafMarkers.remove(path); + if (value != null) { + // marker is surplus + removed.add(value); + if (recordSurplusMarkers) { + surplusMarkers.put(path, value); + } + } + } + + /** + * Get the map of leaf markers. + * @return all leaf markers. + */ + public Map getLeafMarkers() { + return leafMarkers; + } + + /** + * Get the map of surplus markers. + *

+ * Empty if they were not being recorded. + * @return all surplus markers. + */ + public Map getSurplusMarkers() { + return surplusMarkers; + } + + public Path getLastDirChecked() { + return lastDirChecked; + } + + + /** + * How many objects were found. + * @return count + */ + public int getObjectsFound() { + return objectsFound; + } + + public int getScanCount() { + return scanCount; + } + + public int getFilesFound() { + return filesFound; + } + + public int getMarkersFound() { + return markersFound; + } + + @Override + public String toString() { + return "DirMarkerTracker{" + + "leafMarkers=" + leafMarkers.size() + + ", surplusMarkers=" + surplusMarkers.size() + + ", lastDirChecked=" + lastDirChecked + + ", filesFound=" + filesFound + + ", scanCount=" + scanCount + + '}'; + } + + /** + * Scan the surplus marker list and remove from it all where the directory + * policy says "keep". This is useful when auditing + * @param policy policy to use when auditing markers for + * inclusion/exclusion. + * @return list of markers stripped + */ + public List removeAllowedMarkers(DirectoryPolicy policy) { + List removed = new ArrayList<>(); + Iterator> entries = + surplusMarkers.entrySet().iterator(); + while (entries.hasNext()) { + Map.Entry entry = entries.next(); + Path path = entry.getKey(); + if (policy.keepDirectoryMarkers(path)) { + // there's a match + // remove it from the map. + entries.remove(); + LOG.debug("Removing {}", entry.getValue()); + removed.add(path); + } + } + return removed; + } + + /** + * This is a marker entry stored in the map and + * returned as markers are deleted. + */ + public static final class Marker { + /** Path of the marker. */ + private final Path path; + + /** + * Key in the store. + */ + private final String key; + + /** + * The file status of the marker. + */ + private final S3ALocatedFileStatus status; + + private Marker(final Path path, + final String key, + final S3ALocatedFileStatus status) { + this.path = path; + this.key = key; + this.status = status; + } + + public Path getPath() { + return path; + } + + public String getKey() { + return key; + } + + public S3ALocatedFileStatus getStatus() { + return status; + } + + /** + * Get the version ID of the status object; may be null. + * @return a version ID, if known. + */ + public String getVersionId() { + return status.getVersionId(); + } + + @Override + public String toString() { + return "Marker{" + + "path=" + path + + ", key='" + key + '\'' + + ", status=" + status + + '}'; + } + + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java new file mode 100644 index 0000000000000..36dd2e4fd2496 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_AUTHORITATIVE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP; + +/** + * Interface for Directory Marker policies to implement. + */ + +public interface DirectoryPolicy { + + + + /** + * Should a directory marker be retained? + * @param path path a file/directory is being created with. + * @return true if the marker MAY be kept, false if it MUST be deleted. + */ + boolean keepDirectoryMarkers(Path path); + + /** + * Get the marker policy. + * @return policy. + */ + MarkerPolicy getMarkerPolicy(); + + /** + * Describe the policy for marker tools and logs. + * @return description of the current policy. + */ + String describe(); + + /** + * Does a specific path have the relevant option. + * This is to be forwarded from the S3AFileSystem.hasPathCapability + * But only for those capabilities related to markers* + * @param path path + * @param capability capability + * @return true if the capability is supported, false if not + * @throws IllegalArgumentException if the capability is unknown. + */ + boolean hasPathCapability(Path path, String capability); + + /** + * Supported retention policies. + */ + enum MarkerPolicy { + + /** + * Delete markers. + *

+ * This is the classic S3A policy, + */ + Delete(DIRECTORY_MARKER_POLICY_DELETE), + + /** + * Keep markers. + *

+ * This is Not backwards compatible. + */ + Keep(DIRECTORY_MARKER_POLICY_KEEP), + + /** + * Keep markers in authoritative paths only. + *

+ * This is Not backwards compatible within the + * auth paths, but is outside these. + */ + Authoritative(DIRECTORY_MARKER_POLICY_AUTHORITATIVE); + + /** + * The name of the option as allowed in configuration files + * and marker-aware tooling. + */ + private final String optionName; + + MarkerPolicy(final String optionName) { + this.optionName = optionName; + } + + /** + * Get the option name. + * @return name of the option + */ + public String getOptionName() { + return optionName; + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicyImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicyImpl.java new file mode 100644 index 0000000000000..a1aa2580b655a --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicyImpl.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + + +import java.util.EnumSet; +import java.util.Locale; +import java.util.Set; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_DIRECTORY_MARKER_POLICY; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_AUTHORITATIVE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP; +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_AWARE; +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP; +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE; +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP; + +/** + * Implementation of directory policy. + */ +public final class DirectoryPolicyImpl + implements DirectoryPolicy { + + private static final Logger LOG = LoggerFactory.getLogger( + DirectoryPolicyImpl.class); + + /** + * Error string when unable to parse the marker policy option. + */ + public static final String UNKNOWN_MARKER_POLICY = + "Unknown policy in " + + DIRECTORY_MARKER_POLICY + ": "; + + /** + * All available policies. + */ + private static final Set AVAILABLE_POLICIES = + EnumSet.allOf(MarkerPolicy.class); + + /** + * Keep all markers. + */ + public static final DirectoryPolicy KEEP = new DirectoryPolicyImpl( + MarkerPolicy.Keep, (p) -> false); + + /** + * Delete all markers. + */ + public static final DirectoryPolicy DELETE = new DirectoryPolicyImpl( + MarkerPolicy.Delete, (p) -> false); + + /** + * Chosen marker policy. + */ + private final MarkerPolicy markerPolicy; + + /** + * Callback to evaluate authoritativeness of a + * path. + */ + private final Predicate authoritativeness; + + /** + * Constructor. + * @param markerPolicy marker policy + * @param authoritativeness function for authoritativeness + */ + public DirectoryPolicyImpl(final MarkerPolicy markerPolicy, + final Predicate authoritativeness) { + this.markerPolicy = markerPolicy; + this.authoritativeness = authoritativeness; + } + + @Override + public boolean keepDirectoryMarkers(final Path path) { + switch (markerPolicy) { + case Keep: + return true; + case Authoritative: + return authoritativeness.test(path); + case Delete: + default: // which cannot happen + return false; + } + } + + @Override + public MarkerPolicy getMarkerPolicy() { + return markerPolicy; + } + + @Override + public String describe() { + return markerPolicy.getOptionName(); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder( + "DirectoryMarkerRetention{"); + sb.append("policy='").append(markerPolicy.getOptionName()).append('\''); + sb.append('}'); + return sb.toString(); + } + + /** + * Return path policy for store and paths. + * @param path path + * @param capability capability + * @return true if a capability is active + */ + @Override + public boolean hasPathCapability(final Path path, final String capability) { + + switch (capability) { + /* + * Marker policy is dynamically determined for the given path. + */ + case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE: + return true; + + case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP: + return markerPolicy == MarkerPolicy.Keep; + + case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE: + return markerPolicy == MarkerPolicy.Delete; + + case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE: + return markerPolicy == MarkerPolicy.Authoritative; + + case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP: + return keepDirectoryMarkers(path); + + case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE: + return !keepDirectoryMarkers(path); + + default: + throw new IllegalArgumentException("Unknown capability " + capability); + } + } + + /** + * Create/Get the policy for this configuration. + * @param conf config + * @param authoritativeness Callback to evaluate authoritativeness of a + * path. + * @return a policy + */ + public static DirectoryPolicy getDirectoryPolicy( + final Configuration conf, + final Predicate authoritativeness) { + DirectoryPolicy policy; + String option = conf.getTrimmed(DIRECTORY_MARKER_POLICY, + DEFAULT_DIRECTORY_MARKER_POLICY); + switch (option.toLowerCase(Locale.ENGLISH)) { + case DIRECTORY_MARKER_POLICY_DELETE: + // backwards compatible. + LOG.debug("Directory markers will be deleted"); + policy = DELETE; + break; + case DIRECTORY_MARKER_POLICY_KEEP: + LOG.info("Directory markers will be kept"); + policy = KEEP; + break; + case DIRECTORY_MARKER_POLICY_AUTHORITATIVE: + LOG.info("Directory markers will be kept on authoritative" + + " paths"); + policy = new DirectoryPolicyImpl(MarkerPolicy.Authoritative, + authoritativeness); + break; + default: + throw new IllegalArgumentException(UNKNOWN_MARKER_POLICY + option); + } + return policy; + } + + /** + * Enumerate all available policies. + * @return set of the policies. + */ + public static Set availablePolicies() { + return AVAILABLE_POLICIES; + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java index 750aebf500a4b..beb19500812b8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; @@ -43,6 +44,7 @@ import org.apache.hadoop.fs.s3a.s3guard.MetadataStore; import org.apache.hadoop.fs.s3a.s3guard.RenameTracker; import org.apache.hadoop.util.DurationInfo; +import org.apache.hadoop.util.OperationDuration; import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_BLOCK_SIZE; @@ -55,19 +57,31 @@ /** * A parallelized rename operation which updates the metastore in the * process, through whichever {@link RenameTracker} the store provides. + *

* The parallel execution is in groups of size * {@link InternalConstants#RENAME_PARALLEL_LIMIT}; it is only * after one group completes that the next group is initiated. + *

* Once enough files have been copied that they meet the * {@link InternalConstants#MAX_ENTRIES_TO_DELETE} threshold, a delete * is initiated. * If it succeeds, the rename continues with the next group of files. - * + *

* The RenameTracker has the task of keeping the metastore up to date * as the rename proceeds. - * + *

+ * Directory Markers which have child entries are never copied; only those + * which represent empty directories are copied in the rename. + * The {@link DirMarkerTracker} tracks which markers must be copied, and + * which can simply be deleted from the source. + * As a result: rename always purges all non-leaf directory markers from + * the copied tree. This is to ensure that even if a directory tree + * is copied from an authoritative path to a non-authoritative one + * there is never any contamination of the non-auth path with markers. + *

* The rename operation implements the classic HDFS rename policy of * rename(file, dir) renames the file under the directory. + *

* * There is no validation of input and output paths. * Callers are required to themselves verify that destination is not under @@ -183,12 +197,59 @@ private void completeActiveCopies(String reason) throws IOException { /** * Queue an object for deletion. + *

+ * This object will be deleted when the next page of objects to delete + * is posted to S3. Therefore, the COPY must have finished + * before that deletion operation takes place. + * This is managed by: + *
    + *
  1. + * The delete operation only being executed once all active + * copies have completed. + *
    2. + *
  2. + * Only queuing objects here whose copy operation has + * been submitted and so is in that thread pool. + *
    3. + *
+ * This method must only be called from the primary thread. * @param path path to the object * @param key key of the object. + * @param version object version. */ - private void queueToDelete(Path path, String key) { + private void queueToDelete(Path path, String key, String version) { + LOG.debug("Queueing to delete {}", path); pathsToDelete.add(path); - keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); + keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key, version)); + } + + /** + * Queue a list of markers for deletion. + *

+ * no-op if the list is empty. + *

+ * See {@link #queueToDelete(Path, String, String)} for + * details on safe use of this method. + * + * @param markersToDelete markers + */ + private void queueToDelete( + List markersToDelete) { + markersToDelete.forEach(m -> + queueToDelete(m)); + } + + /** + * Queue a single marker for deletion. + *

+ * See {@link #queueToDelete(Path, String, String)} for + * details on safe use of this method. + * + * @param marker markers + */ + private void queueToDelete(final DirMarkerTracker.Marker marker) { + queueToDelete(marker.getPath(), marker.getKey(), + marker.getStatus().getVersionId()); } /** @@ -225,11 +286,19 @@ public Long execute() throws IOException { storeContext, sourcePath, sourceStatus, destPath); + // The path to whichever file or directory is created by the + // rename. When deleting markers all parents of + // this path will need their markers pruned. + Path destCreated = destPath; // Ok! Time to start try { if (sourceStatus.isFile()) { - renameFileToDest(); + // rename the file. The destination path will be different + // from that passed in if the destination is a directory; + // the final value is needed to completely delete parent markers + // when they are not being retained. + destCreated = renameFileToDest(); } else { recursiveDirectoryRename(); } @@ -254,15 +323,17 @@ public Long execute() throws IOException { // Tell the metastore this fact and let it complete its changes renameTracker.completeRename(); - callbacks.finishRename(sourcePath, destPath); + callbacks.finishRename(sourcePath, destCreated); return bytesCopied.get(); } /** - * The source is a file: rename it to the destination. + * The source is a file: rename it to the destination, which + * will be under the current destination path if that is a directory. + * @return the path of the object created. * @throws IOException failure */ - protected void renameFileToDest() throws IOException { + protected Path renameFileToDest() throws IOException { final StoreContext storeContext = getStoreContext(); // the source is a file. Path copyDestinationPath = destPath; @@ -295,12 +366,14 @@ protected void renameFileToDest() throws IOException { callbacks.deleteObjectAtPath(sourcePath, sourceKey, true, null); // and update the tracker renameTracker.sourceObjectsDeleted(Lists.newArrayList(sourcePath)); + return copyDestinationPath; } /** * Execute a full recursive rename. - * The source is a file: rename it to the destination. - * @throws IOException failure + * There is a special handling of directly markers here -only leaf markers + * are copied. This reduces incompatibility "regions" across versions. +Are * @throws IOException failure */ protected void recursiveDirectoryRename() throws IOException { final StoreContext storeContext = getStoreContext(); @@ -325,10 +398,18 @@ protected void recursiveDirectoryRename() throws IOException { // marker. LOG.debug("Deleting fake directory marker at destination {}", destStatus.getPath()); + // Although the dir marker policy doesn't always need to do this, + // it's simplest just to be consistent here. callbacks.deleteObjectAtPath(destStatus.getPath(), dstKey, false, null); } Path parentPath = storeContext.keyToPath(srcKey); + + // Track directory markers so that we know which leaf directories need to be + // recreated + DirMarkerTracker dirMarkerTracker = new DirMarkerTracker(parentPath, + false); + final RemoteIterator iterator = callbacks.listFilesAndEmptyDirectories(parentPath, sourceStatus, @@ -347,36 +428,45 @@ protected void recursiveDirectoryRename() throws IOException { // the source object to copy as a path. Path childSourcePath = storeContext.keyToPath(key); - // mark for deletion on a successful copy. - queueToDelete(childSourcePath, key); - - // the destination key is that of the key under the source tree, - // remapped under the new destination path. - String newDestKey = - dstKey + key.substring(srcKey.length()); - Path childDestPath = storeContext.keyToPath(newDestKey); - - // now begin the single copy - CompletableFuture copy = initiateCopy(child, key, - childSourcePath, newDestKey, childDestPath); - activeCopies.add(copy); - bytesCopied.addAndGet(sourceStatus.getLen()); + List markersToDelete; - if (activeCopies.size() == RENAME_PARALLEL_LIMIT) { - // the limit of active copies has been reached; - // wait for completion or errors to surface. - LOG.debug("Waiting for active copies to complete"); - completeActiveCopies("batch threshold reached"); - } - if (keysToDelete.size() == pageSize) { - // finish ongoing copies then delete all queued keys. - // provided the parallel limit is a factor of the max entry - // constant, this will not need to block for the copy, and - // simply jump straight to the delete. - completeActiveCopiesAndDeleteSources("paged delete"); + boolean isMarker = key.endsWith("/"); + if (isMarker) { + // add the marker to the tracker. + // it will not be deleted _yet_ but it may find a list of parent + // markers which may now be deleted. + markersToDelete = dirMarkerTracker.markerFound( + childSourcePath, key, child); + } else { + // it is a file. + // note that it has been found -this may find a list of parent + // markers which may now be deleted. + markersToDelete = dirMarkerTracker.fileFound( + childSourcePath, key, child); + // the destination key is that of the key under the source tree, + // remapped under the new destination path. + String newDestKey = + dstKey + key.substring(srcKey.length()); + Path childDestPath = storeContext.keyToPath(newDestKey); + + // mark the source file for deletion on a successful copy. + queueToDelete(childSourcePath, key, child.getVersionId()); + // now begin the single copy + CompletableFuture copy = initiateCopy(child, key, + childSourcePath, newDestKey, childDestPath); + activeCopies.add(copy); + bytesCopied.addAndGet(sourceStatus.getLen()); } + // add any markers to delete to the operation so they get cleaned + // incrementally + queueToDelete(markersToDelete); + // and trigger any end of loop operations + endOfLoopActions(); } // end of iteration through the list + // finally process remaining directory markers + copyEmptyDirectoryMarkers(srcKey, dstKey, dirMarkerTracker); + // await the final set of copies and their deletion // This will notify the renameTracker that these objects // have been deleted. @@ -387,6 +477,93 @@ protected void recursiveDirectoryRename() throws IOException { renameTracker.moveSourceDirectory(); } + /** + * Operations to perform at the end of every loop iteration. + *

+ * This may block the thread waiting for copies to complete + * and/or delete a page of data. + */ + private void endOfLoopActions() throws IOException { + if (keysToDelete.size() == pageSize) { + // finish ongoing copies then delete all queued keys. + completeActiveCopiesAndDeleteSources("paged delete"); + } else { + if (activeCopies.size() == RENAME_PARALLEL_LIMIT) { + // the limit of active copies has been reached; + // wait for completion or errors to surface. + LOG.debug("Waiting for active copies to complete"); + completeActiveCopies("batch threshold reached"); + } + } + } + + /** + * Process all directory markers at the end of the rename. + * All leaf markers are queued to be copied in the store; + * this updates the metastore tracker as it does so. + *

+ * Why not simply create new markers? All the metadata + * gets copied too, so if there was anything relevant then + * it would be preserved. + *

+ * At the same time: markers aren't valued much and may + * be deleted without any safety checks -so if there was relevant + * data it is at risk of destruction at any point. + * If there are lots of empty directory rename operations taking place, + * the decision to copy the source may need revisiting. + * Be advised though: the costs of the copy not withstanding, + * it is a lot easier to have one single type of scheduled copy operation + * than have copy and touch calls being scheduled. + *

+ * The duration returned is the time to initiate all copy/delete operations, + * including any blocking waits for active copies and paged deletes + * to execute. There may still be outstanding operations + * queued by this method -the duration may be an underestimate + * of the time this operation actually takes. + * + * @param srcKey source key with trailing / + * @param dstKey dest key with trailing / + * @param dirMarkerTracker tracker of markers + * @return how long it took. + */ + private OperationDuration copyEmptyDirectoryMarkers( + final String srcKey, + final String dstKey, + final DirMarkerTracker dirMarkerTracker) throws IOException { + // directory marker work. + LOG.debug("Copying markers from {}", dirMarkerTracker); + final StoreContext storeContext = getStoreContext(); + Map leafMarkers = + dirMarkerTracker.getLeafMarkers(); + Map surplus = + dirMarkerTracker.getSurplusMarkers(); + // for all leaf markers: copy the original + DurationInfo duration = new DurationInfo(LOG, false, + "copying %d leaf markers with %d surplus not copied", + leafMarkers.size(), surplus.size()); + for (DirMarkerTracker.Marker entry: leafMarkers.values()) { + Path source = entry.getPath(); + String key = entry.getKey(); + String newDestKey = + dstKey + key.substring(srcKey.length()); + Path childDestPath = storeContext.keyToPath(newDestKey); + LOG.debug("copying dir marker from {} to {}", key, newDestKey); + + activeCopies.add( + initiateCopy( + entry.getStatus(), + key, + source, + newDestKey, + childDestPath)); + queueToDelete(entry); + // end of loop + endOfLoopActions(); + } + duration.close(); + return duration; + } + /** * Initiate a copy operation in the executor. * @param source status of the source object. @@ -487,6 +664,16 @@ private void removeSourceObjects( List undeletedObjects = new ArrayList<>(); try { // remove the keys + + // list what is being deleted for the interest of anyone + // who is trying to debug why objects are no longer there. + if (LOG.isDebugEnabled()) { + LOG.debug("Initiating delete operation for {} objects", keys.size()); + for (DeleteObjectsRequest.KeyVersion key : keys) { + LOG.debug(" {} {}", key.getKey(), + key.getVersion() != null ? key.getVersion() : ""); + } + } // this will update the metastore on a failure, but on // a successful operation leaves the store as is. callbacks.removeKeys( @@ -498,7 +685,7 @@ private void removeSourceObjects( // and clear the list. } catch (AmazonClientException | IOException e) { // Failed. - // Notify the rename operation. + // Notify the rename tracker. // removeKeys will have already purged the metastore of // all keys it has known to delete; this is just a final // bit of housekeeping and a chance to tune exception diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StatusProbeEnum.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StatusProbeEnum.java index f843b20ab28b0..3b69c7efe3741 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StatusProbeEnum.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StatusProbeEnum.java @@ -21,9 +21,12 @@ import java.util.EnumSet; import java.util.Set; +import org.apache.hadoop.classification.InterfaceAudience; + /** * Enum of probes which can be made of S3. */ +@InterfaceAudience.Private public enum StatusProbeEnum { /** The actual path. */ @@ -33,28 +36,23 @@ public enum StatusProbeEnum { /** LIST under the path. */ List; - /** All probes. */ - public static final Set ALL = EnumSet.allOf( - StatusProbeEnum.class); - - /** Skip the HEAD and only look for directories. */ - public static final Set DIRECTORIES = - EnumSet.of(DirMarker, List); - - /** We only want the HEAD or dir marker. */ - public static final Set HEAD_OR_DIR_MARKER = - EnumSet.of(Head, DirMarker); + /** Look for files and directories. */ + public static final Set ALL = + EnumSet.of(Head, List); /** We only want the HEAD. */ public static final Set HEAD_ONLY = EnumSet.of(Head); - /** We only want the dir marker. */ - public static final Set DIR_MARKER_ONLY = - EnumSet.of(DirMarker); - - /** We only want the dir marker. */ + /** List operation only. */ public static final Set LIST_ONLY = EnumSet.of(List); + /** Look for files and directories. */ + public static final Set FILE = + HEAD_ONLY; + + /** Skip the HEAD and only look for directories. */ + public static final Set DIRECTORIES = + LIST_ONLY; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java index 213ffdc983718..5f033fa11f834 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -118,6 +119,10 @@ public Collection getListing() { return Collections.unmodifiableCollection(listMap.values()); } + /** + * List all tombstones. + * @return all tombstones in the listing. + */ public Set listTombstones() { Set tombstones = new HashSet<>(); for (PathMetadata meta : listMap.values()) { @@ -128,6 +133,12 @@ public Set listTombstones() { return tombstones; } + /** + * Get the directory listing excluding tombstones. + * Returns a new DirListingMetadata instances, without the tombstones -the + * lastUpdated field is copied from this instance. + * @return a new DirListingMetadata without the tombstones. + */ public DirListingMetadata withoutTombstones() { Collection filteredList = new ArrayList<>(); for (PathMetadata meta : listMap.values()) { @@ -143,6 +154,7 @@ public DirListingMetadata withoutTombstones() { * @return number of entries tracked. This is not the same as the number * of entries in the actual directory unless {@link #isAuthoritative()} is * true. + * It will also include any tombstones. */ public int numEntries() { return listMap.size(); @@ -251,19 +263,24 @@ public String toString() { * Remove expired entries from the listing based on TTL. * @param ttl the ttl time * @param now the current time + * @return the expired values. */ - public synchronized void removeExpiredEntriesFromListing(long ttl, - long now) { + public synchronized List removeExpiredEntriesFromListing( + long ttl, long now) { + List expired = new ArrayList<>(); final Iterator> iterator = listMap.entrySet().iterator(); while (iterator.hasNext()) { final Map.Entry entry = iterator.next(); // we filter iff the lastupdated is not 0 and the entry is expired - if (entry.getValue().getLastUpdated() != 0 - && (entry.getValue().getLastUpdated() + ttl) <= now) { + PathMetadata metadata = entry.getValue(); + if (metadata.getLastUpdated() != 0 + && (metadata.getLastUpdated() + ttl) <= now) { + expired.add(metadata); iterator.remove(); } } + return expired; } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ITtlTimeProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ITtlTimeProvider.java index daee6211b41d9..aa7fc4721b483 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ITtlTimeProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ITtlTimeProvider.java @@ -29,6 +29,19 @@ * Time is measured in milliseconds, */ public interface ITtlTimeProvider { + + /** + * The current time in milliseconds. + * Assuming this calls System.currentTimeMillis(), this is a native iO call + * and so should be invoked sparingly (i.e. evaluate before any loop, rather + * than inside). + * @return the current time. + */ long getNow(); + + /** + * The TTL of the metadata. + * @return time in millis after which metadata is considered out of date. + */ long getMetadataTtl(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java index 666c233575ad6..722f42176ef2f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java @@ -172,8 +172,10 @@ private static final class NullRenameTracker extends RenameTracker { private NullRenameTracker( final StoreContext storeContext, final Path source, - final Path dest, MetadataStore metadataStore) { - super("null tracker", storeContext, metadataStore, source, dest, null); + final Path dest, + MetadataStore metadataStore) { + super("NullRenameTracker", storeContext, metadataStore, source, dest, + null); } @Override diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java index 05ebe671662ea..ae5c293d639ff 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java @@ -159,6 +159,54 @@ static Class getMetadataStoreClass( } + /** + * We update the metastore for the specific case of S3 value == S3Guard value + * so as to place a more recent modtime in the store. + * because if not, we will continue to probe S3 whenever we look for this + * object, even we only do this if confident the S3 status is the same + * as the one in the store (i.e. it is not an older version) + * @param metadataStore MetadataStore to {@code put()} into. + * @param pm current data + * @param s3AFileStatus status to store + * @param timeProvider Time provider to use when writing entries + * @return true if the entry was updated. + * @throws IOException if metadata store update failed + */ + @RetryTranslated + public static boolean refreshEntry( + MetadataStore metadataStore, + PathMetadata pm, + S3AFileStatus s3AFileStatus, + ITtlTimeProvider timeProvider) throws IOException { + // the modtime of the data is the same as/older than the s3guard value + // either an old object has been found, or the existing one was retrieved + // in both cases -return s3guard value + S3AFileStatus msStatus = pm.getFileStatus(); + + // first check: size + boolean sizeMatch = msStatus.getLen() == s3AFileStatus.getLen(); + + // etags are expected on all objects, but handle the situation + // that a third party store doesn't serve them. + String s3Etag = s3AFileStatus.getETag(); + String pmEtag = msStatus.getETag(); + boolean etagsMatch = s3Etag != null && s3Etag.equals(pmEtag); + + // version ID: only in some stores, and will be missing in the metastore + // if the entry was created through a list operation. + String s3VersionId = s3AFileStatus.getVersionId(); + String pmVersionId = msStatus.getVersionId(); + boolean versionsMatchOrMissingInMetastore = + pmVersionId == null || pmVersionId.equals(s3VersionId); + if (sizeMatch && etagsMatch && versionsMatchOrMissingInMetastore) { + // update the store, return the new value + LOG.debug("Refreshing the metastore entry/timestamp"); + putAndReturn(metadataStore, s3AFileStatus, timeProvider); + return true; + } + return false; + } + /** * Helper function which puts a given S3AFileStatus into the MetadataStore and * returns the same S3AFileStatus. Instrumentation monitors the put operation. @@ -314,14 +362,14 @@ public static S3AFileStatus[] dirMetaToStatuses(DirListingMetadata dirMeta) { * @return Final result of directory listing. * @throws IOException if metadata store update failed */ - public static FileStatus[] dirListingUnion(MetadataStore ms, Path path, + public static S3AFileStatus[] dirListingUnion(MetadataStore ms, Path path, List backingStatuses, DirListingMetadata dirMeta, boolean isAuthoritative, ITtlTimeProvider timeProvider) throws IOException { // Fast-path for NullMetadataStore if (isNullMetadataStore(ms)) { - return backingStatuses.toArray(new FileStatus[backingStatuses.size()]); + return backingStatuses.toArray(new S3AFileStatus[backingStatuses.size()]); } assertQualified(path); @@ -927,8 +975,10 @@ public static PathMetadata getWithTtl(MetadataStore ms, Path path, if (!pathMetadata.isExpired(ttl, timeProvider.getNow())) { return pathMetadata; } else { - LOG.debug("PathMetadata TTl for {} is expired in metadata store.", - path); + LOG.debug("PathMetadata TTl for {} is expired in metadata store" + + " -removing entry", path); + // delete the tombstone + ms.forgetMetadata(path); return null; } } @@ -940,6 +990,8 @@ public static PathMetadata getWithTtl(MetadataStore ms, Path path, * List children; mark the result as non-auth if the TTL has expired. * If the allowAuthoritative flag is true, return without filtering or * checking for TTL expiry. + * If false: the expiry scan takes place and the + * TODO: should we always purge tombstones? Even in auth? * @param ms metastore * @param path path to look up. * @param timeProvider nullable time provider @@ -968,9 +1020,15 @@ public static DirListingMetadata listChildrenWithTtl(MetadataStore ms, // filter expired entries if (dlm != null) { - dlm.removeExpiredEntriesFromListing( + List expired = dlm.removeExpiredEntriesFromListing( timeProvider.getMetadataTtl(), timeProvider.getNow()); + // now purge the tombstones + for (PathMetadata metadata : expired) { + if (metadata.isDeleted()) { + ms.forgetMetadata(metadata.getFileStatus().getPath()); + } + } } return dlm; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 6e89d0cd2dadb..f89777f730376 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -30,12 +30,14 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import com.amazonaws.services.s3.model.MultipartUpload; import com.google.common.annotations.VisibleForTesting; @@ -44,12 +46,15 @@ import org.slf4j.LoggerFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.s3a.MultipartUtils; import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3AFileSystem; @@ -58,7 +63,10 @@ import org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens; import org.apache.hadoop.fs.s3a.commit.CommitConstants; import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants; +import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; +import org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl; import org.apache.hadoop.fs.s3a.select.SelectTool; +import org.apache.hadoop.fs.s3a.tools.MarkerTool; import org.apache.hadoop.fs.shell.CommandFormat; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -79,7 +87,11 @@ /** * CLI to manage S3Guard Metadata Store. + *

+ * Some management tools invoke this class directly. */ +@InterfaceAudience.LimitedPrivate("management tools") +@InterfaceStability.Evolving public abstract class S3GuardTool extends Configured implements Tool, Closeable { private static final Logger LOG = LoggerFactory.getLogger(S3GuardTool.class); @@ -98,15 +110,17 @@ public abstract class S3GuardTool extends Configured implements Tool, "Commands: \n" + "\t" + Init.NAME + " - " + Init.PURPOSE + "\n" + "\t" + Destroy.NAME + " - " + Destroy.PURPOSE + "\n" + - "\t" + Import.NAME + " - " + Import.PURPOSE + "\n" + + "\t" + Authoritative.NAME + " - " + Authoritative.PURPOSE + "\n" + "\t" + BucketInfo.NAME + " - " + BucketInfo.PURPOSE + "\n" + - "\t" + Uploads.NAME + " - " + Uploads.PURPOSE + "\n" + "\t" + Diff.NAME + " - " + Diff.PURPOSE + "\n" + + "\t" + Fsck.NAME + " - " + Fsck.PURPOSE + "\n" + + "\t" + Import.NAME + " - " + Import.PURPOSE + "\n" + + "\t" + MarkerTool.MARKERS + " - " + MarkerTool.PURPOSE + "\n" + "\t" + Prune.NAME + " - " + Prune.PURPOSE + "\n" + "\t" + SetCapacity.NAME + " - " + SetCapacity.PURPOSE + "\n" + "\t" + SelectTool.NAME + " - " + SelectTool.PURPOSE + "\n" + - "\t" + Fsck.NAME + " - " + Fsck.PURPOSE + "\n" + - "\t" + Authoritative.NAME + " - " + Authoritative.PURPOSE + "\n"; + "\t" + Uploads.NAME + " - " + Uploads.PURPOSE + "\n"; + private static final String DATA_IN_S3_IS_PRESERVED = "(all data in S3 is preserved)"; @@ -116,6 +130,7 @@ public abstract class S3GuardTool extends Configured implements Tool, static final int SUCCESS = EXIT_SUCCESS; static final int INVALID_ARGUMENT = EXIT_COMMAND_ARGUMENT_ERROR; static final int E_USAGE = EXIT_USAGE; + static final int ERROR = EXIT_FAIL; static final int E_BAD_STATE = EXIT_NOT_ACCEPTABLE; static final int E_NOT_FOUND = EXIT_NOT_FOUND; @@ -472,6 +487,14 @@ protected void setStore(MetadataStore store) { this.store = store; } + /** + * Reset the store and filesystem bindings. + */ + protected void resetBindings() { + store = null; + filesystem = null; + } + protected CommandFormat getCommandFormat() { return commandFormat; } @@ -497,6 +520,30 @@ public final int run(String[] args) throws Exception { public abstract int run(String[] args, PrintStream out) throws Exception, ExitUtil.ExitException; + /** + * Dump the filesystem Storage Statistics if the FS is not null. + * Only non-zero statistics are printed. + * @param stream output stream + */ + protected void dumpFileSystemStatistics(PrintStream stream) { + FileSystem fs = getFilesystem(); + if (fs == null) { + return; + } + println(stream, "%nStorage Statistics for %s%n", fs.getUri()); + StorageStatistics st = fs.getStorageStatistics(); + Iterator it + = st.getLongStatistics(); + while (it.hasNext()) { + StorageStatistics.LongStatistic next = it.next(); + long value = next.getValue(); + if (value != 0) { + println(stream, "%s\t%s", next.getName(), value); + } + } + println(stream, ""); + } + /** * Create the metadata store. */ @@ -1167,16 +1214,20 @@ public int run(String[] args, PrintStream out) throws * Get info about a bucket and its S3Guard integration status. */ public static class BucketInfo extends S3GuardTool { - public static final String NAME = "bucket-info"; + public static final String BUCKET_INFO = "bucket-info"; + public static final String NAME = BUCKET_INFO; public static final String GUARDED_FLAG = "guarded"; public static final String UNGUARDED_FLAG = "unguarded"; public static final String AUTH_FLAG = "auth"; public static final String NONAUTH_FLAG = "nonauth"; public static final String ENCRYPTION_FLAG = "encryption"; public static final String MAGIC_FLAG = "magic"; + public static final String MARKERS_FLAG = "markers"; + public static final String MARKERS_AWARE = "aware"; public static final String PURPOSE = "provide/check S3Guard information" + " about a specific bucket"; + private static final String USAGE = NAME + " [OPTIONS] s3a://BUCKET\n" + "\t" + PURPOSE + "\n\n" + "Common options:\n" @@ -1186,7 +1237,9 @@ public static class BucketInfo extends S3GuardTool { + " -" + NONAUTH_FLAG + " - Require the S3Guard mode to be \"non-authoritative\"\n" + " -" + MAGIC_FLAG + " - Require the S3 filesystem to be support the \"magic\" committer\n" + " -" + ENCRYPTION_FLAG - + " -require {none, sse-s3, sse-kms} - Require encryption policy"; + + " (none, sse-s3, sse-kms) - Require encryption policy\n" + + " -" + MARKERS_FLAG + + " (aware, keep, delete, authoritative) - directory markers policy\n"; /** * Output when the client cannot get the location of a bucket. @@ -1196,10 +1249,17 @@ public static class BucketInfo extends S3GuardTool { "Location unknown -caller lacks " + RolePolicies.S3_GET_BUCKET_LOCATION + " permission"; + + @VisibleForTesting + public static final String IS_MARKER_AWARE = + "The S3A connector is compatible with buckets where" + + " directory markers are not deleted"; + public BucketInfo(Configuration conf) { super(conf, GUARDED_FLAG, UNGUARDED_FLAG, AUTH_FLAG, NONAUTH_FLAG, MAGIC_FLAG); CommandFormat format = getCommandFormat(); format.addOptionWithValue(ENCRYPTION_FLAG); + format.addOptionWithValue(MARKERS_FLAG); } @Override @@ -1384,10 +1444,57 @@ public int run(String[] args, PrintStream out) fsUri, desiredEncryption, encryption); } + // directory markers + processMarkerOption(out, fs, + getCommandFormat().getOptValue(MARKERS_FLAG)); + + // and finally flush the output and report a success. out.flush(); return SUCCESS; } + /** + * Validate the marker options. + * @param out output stream + * @param fs filesystem + * @param path test path + * @param marker desired marker option -may be null. + */ + private void processMarkerOption(final PrintStream out, + final S3AFileSystem fs, + final String marker) { + DirectoryPolicy markerPolicy = fs.getDirectoryMarkerPolicy(); + String desc = markerPolicy.describe(); + println(out, "%nThe directory marker policy is \"%s\"%n", desc); + + DirectoryPolicy.MarkerPolicy mp = markerPolicy.getMarkerPolicy(); + + String desiredMarker = marker == null + ? "" + : marker.trim(); + final String optionName = mp.getOptionName(); + if (!desiredMarker.isEmpty()) { + if (MARKERS_AWARE.equalsIgnoreCase(desiredMarker)) { + // simple awareness test -provides a way to validate compatibility + // on the command line + println(out, IS_MARKER_AWARE); + String pols = DirectoryPolicyImpl.availablePolicies() + .stream() + .map(DirectoryPolicy.MarkerPolicy::getOptionName) + .collect(Collectors.joining(", ")); + println(out, "Available Policies: %s", pols); + + } else { + // compare with current policy + if (!optionName.equalsIgnoreCase(desiredMarker)) { + throw badState("Bucket %s: required marker policy is \"%s\"" + + " but actual policy is \"%s\"", + fs.getUri(), desiredMarker, optionName); + } + } + } + } + private String printOption(PrintStream out, String description, String key, String defVal) { String t = getFilesystem().getConf().getTrimmed(key, defVal); @@ -1991,6 +2098,9 @@ public static int run(Configuration conf, String...args) throws case Diff.NAME: command = new Diff(conf); break; + case MarkerTool.MARKERS: + command = new MarkerTool(conf); + break; case Prune.NAME: command = new Prune(conf); break; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java new file mode 100644 index 0000000000000..6855c52edbbbb --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java @@ -0,0 +1,723 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.tools; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintStream; +import java.io.Writer; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.MultiObjectDeleteException; +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.Retries; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus; +import org.apache.hadoop.fs.s3a.UnknownStoreException; +import org.apache.hadoop.fs.s3a.impl.DirMarkerTracker; +import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; +import org.apache.hadoop.fs.s3a.impl.StoreContext; +import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool; +import org.apache.hadoop.fs.shell.CommandFormat; +import org.apache.hadoop.util.DurationInfo; +import org.apache.hadoop.util.ExitUtil; + +import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH; +import static org.apache.hadoop.fs.s3a.Constants.BULK_DELETE_PAGE_SIZE; +import static org.apache.hadoop.fs.s3a.Constants.BULK_DELETE_PAGE_SIZE_DEFAULT; +import static org.apache.hadoop.fs.s3a.Invoker.once; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_INTERRUPTED; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_ACCEPTABLE; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_FOUND; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_SUCCESS; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE; + +/** + * Audit and S3 bucket for directory markers. + *

+ * This tool does not go anywhere near S3Guard; its scan bypasses any + * metastore as we are explicitly looking for marker objects. + */ +@InterfaceAudience.LimitedPrivate("management tools") +@InterfaceStability.Unstable +public final class MarkerTool extends S3GuardTool { + + private static final Logger LOG = LoggerFactory.getLogger(MarkerTool.class); + + /** + * Name of this tool: {@value}. + */ + public static final String MARKERS = "markers"; + + /** + * Purpose of this tool: {@value}. + */ + public static final String PURPOSE = + "View and manipulate S3 directory markers"; + + /** + * Audit sub-command: {@value}. + */ + public static final String OPT_AUDIT = "audit"; + + /** + * Clean Sub-command: {@value}. + */ + public static final String OPT_CLEAN = "clean"; + + /** + * Audit sub-command: {@value}. + */ + public static final String AUDIT = "-" + OPT_AUDIT; + + /** + * Clean Sub-command: {@value}. + */ + public static final String CLEAN = "-" + OPT_CLEAN; + + /** + * Expected number of markers to find: {@value}. + */ + public static final String OPT_EXPECTED = "expected"; + + /** + * Name of a file to save the list of markers to: {@value}. + */ + public static final String OPT_OUT = "out"; + + /** + * Limit of objects to scan: {@value}. + */ + public static final String OPT_LIMIT = "limit"; + + /** + * Only consider markers found in non-authoritative paths + * as failures: {@value}. + */ + public static final String OPT_NONAUTH = "nonauth"; + + /** + * Error text when too few arguments are found. + */ + @VisibleForTesting + static final String E_ARGUMENTS = "Wrong number of arguments: %d"; + + /** + * Constant to use when there is no limit on the number of + * objects listed: {@value}. + *

+ * The value is 0 and not -1 because it allows for the limit to be + * set on the command line {@code -limit 0}. + * The command line parser rejects {@code -limit -1} as the -1 + * is interpreted as the (unknown) option "-1". + */ + public static final int UNLIMITED_LISTING = 0; + + + /** + * Usage string: {@value}. + */ + private static final String USAGE = MARKERS + + " (-" + OPT_AUDIT + + " | -" + OPT_CLEAN + ")" + + " [-" + OPT_EXPECTED + " ]" + + " [-" + OPT_OUT + " ]" + + " [-" + OPT_LIMIT + " ]" + + " [-" + OPT_NONAUTH + "]" + + " [-" + VERBOSE + "]" + + + " \n" + + "\t" + PURPOSE + "\n\n"; + + /** Will be overridden in run(), but during tests needs to avoid NPEs. */ + private PrintStream out = System.out; + + /** + * Verbosity flag. + */ + private boolean verbose; + + /** + * Store context. + */ + private StoreContext storeContext; + + /** + * Operations during the scan. + */ + private MarkerToolOperations operations; + + /** + * Constructor. + * @param conf configuration + */ + public MarkerTool(final Configuration conf) { + super(conf, + OPT_AUDIT, + OPT_CLEAN, + VERBOSE, + OPT_NONAUTH); + CommandFormat format = getCommandFormat(); + format.addOptionWithValue(OPT_EXPECTED); + format.addOptionWithValue(OPT_LIMIT); + format.addOptionWithValue(OPT_OUT); + } + + @Override + public String getUsage() { + return USAGE; + } + + @Override + public String getName() { + return MARKERS; + } + + @Override + public void resetBindings() { + super.resetBindings(); + storeContext = null; + operations = null; + } + + @Override + public int run(final String[] args, final PrintStream stream) + throws ExitUtil.ExitException, Exception { + this.out = stream; + final List parsedArgs; + try { + parsedArgs = parseArgs(args); + } catch (CommandFormat.UnknownOptionException e) { + errorln(getUsage()); + throw new ExitUtil.ExitException(EXIT_USAGE, e.getMessage(), e); + } + if (parsedArgs.size() != 1) { + errorln(getUsage()); + println(out, "Supplied arguments: [" + + parsedArgs.stream() + .collect(Collectors.joining(", ")) + + "]"); + throw new ExitUtil.ExitException(EXIT_USAGE, + String.format(E_ARGUMENTS, parsedArgs.size())); + } + // read arguments + CommandFormat command = getCommandFormat(); + verbose = command.getOpt(VERBOSE); + + // How many markers are expected? + int expected = 0; + String value = command.getOptValue(OPT_EXPECTED); + if (value != null && !value.isEmpty()) { + expected = Integer.parseInt(value); + } + + // determine the action + boolean audit = command.getOpt(OPT_AUDIT); + boolean clean = command.getOpt(OPT_CLEAN); + if (audit == clean) { + // either both are set or neither are set + // this is equivalent to (not audit xor clean) + errorln(getUsage()); + throw new ExitUtil.ExitException(EXIT_USAGE, + "Exactly one of " + AUDIT + " and " + CLEAN); + } + int limit = UNLIMITED_LISTING; + value = command.getOptValue(OPT_LIMIT); + if (value != null && !value.isEmpty()) { + limit = Integer.parseInt(value); + } + final String dir = parsedArgs.get(0); + Path path = new Path(dir); + URI uri = path.toUri(); + if (uri.getPath().isEmpty()) { + // fix up empty URI for better CLI experience + path = new Path(path, "/"); + } + FileSystem fs = path.getFileSystem(getConf()); + ScanResult result = execute( + fs, + path, + clean, + expected, + limit, + command.getOpt(OPT_NONAUTH)); + if (verbose) { + dumpFileSystemStatistics(out); + } + + // and finally see if the output should be saved to a file + String saveFile = command.getOptValue(OPT_OUT); + if (saveFile != null && !saveFile.isEmpty()) { + println(out, "Saving result to %s", saveFile); + try (Writer writer = + new OutputStreamWriter( + new FileOutputStream(saveFile), + StandardCharsets.UTF_8)) { + final List surplus = result.getTracker() + .getSurplusMarkers() + .keySet() + .stream() + .map(p-> p.toString() + "/") + .sorted() + .collect(Collectors.toList()); + IOUtils.writeLines(surplus, "\n", writer); + } + } + return result.exitCode; + } + + /** + * Execute the scan/purge. + * @param sourceFS source FS; must be or wrap an S3A FS. + * @param path path to scan. + * @param doPurge purge? + * @param expectedMarkerCount expected marker count + * @param limit limit of files to scan; -1 for 'unlimited' + * @param nonAuth consider only markers in nonauth paths as errors + * @return scan+purge result. + * @throws IOException failure + */ + @VisibleForTesting + ScanResult execute( + final FileSystem sourceFS, + final Path path, + final boolean doPurge, + final int expectedMarkerCount, + final int limit, + final boolean nonAuth) + throws IOException { + S3AFileSystem fs = bindFilesystem(sourceFS); + + // extract the callbacks needed for the rest of the work + storeContext = fs.createStoreContext(); + operations = fs.createMarkerToolOperations(); + // filesystem policy. + // if the -nonauth option is set, this is used to filter + // out surplus markers from the results. + DirectoryPolicy activePolicy = fs.getDirectoryMarkerPolicy(); + DirectoryPolicy.MarkerPolicy policy = activePolicy + .getMarkerPolicy(); + println(out, "The directory marker policy of %s is \"%s\"", + storeContext.getFsURI(), + policy); + String authPath = storeContext.getConfiguration() + .getTrimmed(AUTHORITATIVE_PATH, ""); + if (policy == DirectoryPolicy.MarkerPolicy.Authoritative) { + // in auth mode, note the auth paths. + println(out, "Authoritative path list is \"%s\"", authPath); + } + // qualify the path + Path target = path.makeQualified(fs.getUri(), new Path("/")); + // initial safety check: does the path exist? + try { + getFilesystem().getFileStatus(target); + } catch (UnknownStoreException ex) { + // bucket doesn't exist. + // replace the stack trace with an error code. + throw new ExitUtil.ExitException(EXIT_NOT_FOUND, + ex.toString(), ex); + + } catch (FileNotFoundException ex) { + throw new ExitUtil.ExitException(EXIT_NOT_FOUND, + "Not found: " + target, ex); + } + + // the default filter policy is that all entries should be deleted + DirectoryPolicy filterPolicy = nonAuth + ? activePolicy + : null; + ScanResult result = scan(target, doPurge, expectedMarkerCount, limit, + filterPolicy); + return result; + } + + /** + * Result of the scan operation. + */ + public static final class ScanResult { + + /** + * Exit code to return if an exception was not raised. + */ + private int exitCode; + + /** + * The tracker. + */ + private DirMarkerTracker tracker; + + /** + * Scan summary. + */ + private MarkerPurgeSummary purgeSummary; + + private ScanResult() { + } + + @Override + public String toString() { + return "ScanResult{" + + "exitCode=" + exitCode + + ", tracker=" + tracker + + ", purgeSummary=" + purgeSummary + + '}'; + } + + /** Exit code to report. */ + public int getExitCode() { + return exitCode; + } + + /** Tracker which did the scan. */ + public DirMarkerTracker getTracker() { + return tracker; + } + + /** Summary of purge. Null if none took place. */ + public MarkerPurgeSummary getPurgeSummary() { + return purgeSummary; + } + } + + /** + * Do the scan/purge. + * @param path path to scan. + * @param clean purge? + * @param expectedMarkerCount expected marker count + * @param limit limit of files to scan; 0 for 'unlimited' + * @param filterPolicy filter policy on a nonauth scan; may be null + * @return result. + * @throws IOException IO failure + * @throws ExitUtil.ExitException explicitly raised failure + */ + @Retries.RetryTranslated + private ScanResult scan( + final Path path, + final boolean clean, + final int expectedMarkerCount, + final int limit, + final DirectoryPolicy filterPolicy) + throws IOException, ExitUtil.ExitException { + + ScanResult result = new ScanResult(); + + // Mission Accomplished + result.exitCode = EXIT_SUCCESS; + // Now do the work. + DirMarkerTracker tracker = new DirMarkerTracker(path, true); + result.tracker = tracker; + boolean completed; + try (DurationInfo ignored = + new DurationInfo(LOG, "marker scan %s", path)) { + completed = scanDirectoryTree(path, tracker, limit); + } + int objectsFound = tracker.getObjectsFound(); + println(out, "Listed %d object%s under %s%n", + objectsFound, + suffix(objectsFound), + path); + // scan done. what have we got? + Map surplusMarkers + = tracker.getSurplusMarkers(); + Map leafMarkers + = tracker.getLeafMarkers(); + int surplus = surplusMarkers.size(); + if (surplus == 0) { + println(out, "No surplus directory markers were found under %s", path); + } else { + println(out, "Found %d surplus directory marker%s under %s", + surplus, + suffix(surplus), + path); + + for (Path markers : surplusMarkers.keySet()) { + println(out, " %s/", markers); + } + } + if (!leafMarkers.isEmpty()) { + println(out, "Found %d empty directory 'leaf' marker%s under %s", + leafMarkers.size(), + suffix(leafMarkers.size()), + path); + for (Path markers : leafMarkers.keySet()) { + println(out, " %s/", markers); + } + println(out, "These are required to indicate empty directories"); + } + + if (clean) { + // clean: remove the markers, do not worry about their + // presence when reporting success/failiure + int deletePageSize = storeContext.getConfiguration() + .getInt(BULK_DELETE_PAGE_SIZE, + BULK_DELETE_PAGE_SIZE_DEFAULT); + result.purgeSummary = purgeMarkers(tracker, deletePageSize); + } else { + // this is an audit, so validate the marker count + + if (filterPolicy != null) { + // if a filter policy is supplied, filter out all markers + // under the auth path + List allowed = tracker.removeAllowedMarkers(filterPolicy); + int allowedMarkers = allowed.size(); + println(out, "%nIgnoring %d marker%s in authoritative paths", + allowedMarkers, suffix(allowedMarkers)); + if (verbose) { + allowed.forEach(p -> println(out, p.toString())); + } + // recalculate the marker size + surplus = surplusMarkers.size(); + } + if (surplus > expectedMarkerCount) { + // failure + if (expectedMarkerCount > 0) { + println(out, "Expected %d marker%s", expectedMarkerCount, + suffix(surplus)); + } + println(out, "Surplus markers were found -failing audit"); + + result.exitCode = EXIT_NOT_ACCEPTABLE; + } + } + + // now one little check for whether a limit was reached. + if (!completed) { + println(out, "Listing limit reached before completing the scan"); + result.exitCode = EXIT_INTERRUPTED; + } + return result; + } + + /** + * Suffix for plurals. + * @param size size to generate a suffix for + * @return "" or "s", depending on size + */ + private String suffix(final int size) { + return size == 1 ? "" : "s"; + } + + /** + * Scan a directory tree. + * @param path path to scan + * @param tracker tracker to update + * @param limit limit of files to scan; -1 for 'unlimited' + * @return true if the scan completedly scanned the entire tree + * @throws IOException IO failure + */ + @Retries.RetryTranslated + private boolean scanDirectoryTree( + final Path path, + final DirMarkerTracker tracker, + final int limit) throws IOException { + + int count = 0; + RemoteIterator listing = operations + .listObjects(path, storeContext.pathToKey(path)); + while (listing.hasNext()) { + count++; + S3AFileStatus status = listing.next(); + Path statusPath = status.getPath(); + S3ALocatedFileStatus locatedStatus = new S3ALocatedFileStatus( + status, null); + String key = storeContext.pathToKey(statusPath); + if (status.isDirectory()) { + if (verbose) { + println(out, " Directory Marker %s/", key); + } + LOG.debug("{}", key); + tracker.markerFound(statusPath, + key + "/", + locatedStatus); + } else { + tracker.fileFound(statusPath, + key, + locatedStatus); + } + if ((count % 1000) == 0) { + println(out, "Scanned %,d objects", count); + } + if (limit > 0 && count >= limit) { + println(out, "Limit of scan reached - %,d object%s", + limit, suffix(limit)); + return false; + } + } + return true; + } + + /** + * Result of a call of {@link #purgeMarkers(DirMarkerTracker, int)}; + * included in {@link ScanResult} so must share visibility. + */ + static final class MarkerPurgeSummary { + + /** Number of markers deleted. */ + private int markersDeleted; + + /** Number of delete requests issued. */ + private int deleteRequests; + + /** + * Total duration of delete requests. + * If this is ever parallelized, this will + * be greater than the elapsed time of the + * operation. + */ + private long totalDeleteRequestDuration; + + @Override + public String toString() { + return "MarkerPurgeSummary{" + + "markersDeleted=" + markersDeleted + + ", deleteRequests=" + deleteRequests + + ", totalDeleteRequestDuration=" + totalDeleteRequestDuration + + '}'; + } + + + int getMarkersDeleted() { + return markersDeleted; + } + + int getDeleteRequests() { + return deleteRequests; + } + + long getTotalDeleteRequestDuration() { + return totalDeleteRequestDuration; + } + } + + /** + * Purge the markers. + * @param tracker tracker with the details + * @param deletePageSize page size of deletes + * @return summary + * @throws IOException IO failure + */ + @Retries.RetryTranslated + private MarkerPurgeSummary purgeMarkers( + final DirMarkerTracker tracker, + final int deletePageSize) + throws MultiObjectDeleteException, AmazonClientException, IOException { + + MarkerPurgeSummary summary = new MarkerPurgeSummary(); + // we get a map of surplus markers to delete. + Map markers + = tracker.getSurplusMarkers(); + int size = markers.size(); + // build a list from the strings in the map + List collect = + markers.values().stream() + .map(p -> new DeleteObjectsRequest.KeyVersion(p.getKey())) + .collect(Collectors.toList()); + // build an array list for ease of creating the lists of + // keys in each page through the subList() method. + List markerKeys = + new ArrayList<>(collect); + + // now randomize. Why so? if the list spans multiple S3 partitions, + // it should reduce the IO load on each part. + Collections.shuffle(markerKeys); + int pages = size / deletePageSize; + if (size % deletePageSize > 0) { + pages += 1; + } + if (verbose) { + println(out, "%n%d marker%s to delete in %d page%s of %d keys/page", + size, suffix(size), + pages, suffix(pages), + deletePageSize); + } + DurationInfo durationInfo = new DurationInfo(LOG, "Deleting markers"); + int start = 0; + while (start < size) { + // end is one past the end of the page + int end = Math.min(start + deletePageSize, size); + List page = markerKeys.subList(start, + end); + List undeleted = new ArrayList<>(); + once("Remove S3 Keys", + tracker.getBasePath().toString(), () -> + operations.removeKeys(page, true, undeleted, null, false)); + summary.deleteRequests++; + // and move to the start of the next page + start = end; + } + durationInfo.close(); + summary.totalDeleteRequestDuration = durationInfo.value(); + summary.markersDeleted = size; + return summary; + } + + public boolean isVerbose() { + return verbose; + } + + public void setVerbose(final boolean verbose) { + this.verbose = verbose; + } + + /** + * Execute the marker tool, with no checks on return codes. + * + * @param sourceFS filesystem to use + * @param path path to scan + * @param doPurge should markers be purged + * @param expectedMarkers number of markers expected + * @param limit limit of files to scan; -1 for 'unlimited' + * @param nonAuth only use nonauth path count for failure rules + * @return the result + */ + @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") + public static MarkerTool.ScanResult execMarkerTool( + final FileSystem sourceFS, + final Path path, + final boolean doPurge, + final int expectedMarkers, + final int limit, boolean nonAuth) throws IOException { + MarkerTool tool = new MarkerTool(sourceFS.getConf()); + tool.setVerbose(LOG.isDebugEnabled()); + + return tool.execute(sourceFS, path, doPurge, + expectedMarkers, limit, nonAuth); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java new file mode 100644 index 0000000000000..9ab7636d6c99f --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.tools; + +import java.io.IOException; +import java.util.List; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.MultiObjectDeleteException; + +import org.apache.hadoop.fs.InvalidRequestException; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.Retries; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState; + +/** + * Operations which must be offered by the store for {@link MarkerTool}. + * These are a proper subset of {@code OperationCallbacks}; this interface + * strips down those provided to the tool. + */ +public interface MarkerToolOperations { + + /** + * Create an iterator over objects in S3 only; S3Guard + * is not involved. + * The listing includes the key itself, if found. + * @param path path of the listing. + * @param key object key + * @return iterator with the first listing completed. + * @throws IOException failure. + */ + @Retries.RetryTranslated + RemoteIterator listObjects( + Path path, + String key) + throws IOException; + + /** + * Remove keys from the store, updating the metastore on a + * partial delete represented as a MultiObjectDeleteException failure by + * deleting all those entries successfully deleted and then rethrowing + * the MultiObjectDeleteException. + * @param keysToDelete collection of keys to delete on the s3-backend. + * if empty, no request is made of the object store. + * @param deleteFakeDir indicates whether this is for deleting fake dirs. + * @param undeletedObjectsOnFailure List which will be built up of all + * files that were not deleted. This happens even as an exception + * is raised. + * @param operationState bulk operation state + * @param quiet should a bulk query be quiet, or should its result list + * all deleted keys + * @return the deletion result if a multi object delete was invoked + * and it returned without a failure, else null. + * @throws InvalidRequestException if the request was rejected due to + * a mistaken attempt to delete the root directory. + * @throws MultiObjectDeleteException one or more of the keys could not + * be deleted in a multiple object delete operation. + * @throws AmazonClientException amazon-layer failure. + * @throws IOException other IO Exception. + */ + @Retries.RetryMixed + DeleteObjectsResult removeKeys( + List keysToDelete, + boolean deleteFakeDir, + List undeletedObjectsOnFailure, + BulkOperationState operationState, + boolean quiet) + throws MultiObjectDeleteException, AmazonClientException, + IOException; + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java new file mode 100644 index 0000000000000..d14bb6b1d8ebb --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.tools; + +import java.io.IOException; +import java.util.List; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.MultiObjectDeleteException; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.impl.OperationCallbacks; +import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState; + +/** + * Implement the marker tool operations by forwarding to the + * {@link OperationCallbacks} instance provided in the constructor. + */ +public class MarkerToolOperationsImpl implements MarkerToolOperations { + + private final OperationCallbacks operationCallbacks; + + /** + * Constructor. + * @param operations implementation of the operations + */ + public MarkerToolOperationsImpl(final OperationCallbacks operations) { + this.operationCallbacks = operations; + } + + @Override + public RemoteIterator listObjects(final Path path, + final String key) + throws IOException { + return operationCallbacks.listObjects(path, key); + } + + @Override + public DeleteObjectsResult removeKeys( + final List keysToDelete, + final boolean deleteFakeDir, + final List undeletedObjectsOnFailure, + final BulkOperationState operationState, + final boolean quiet) + throws MultiObjectDeleteException, AmazonClientException, IOException { + return operationCallbacks.removeKeys(keysToDelete, deleteFakeDir, + undeletedObjectsOnFailure, operationState, quiet); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/package-info.java new file mode 100644 index 0000000000000..cb3a3749b658c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/package-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * S3A Command line tools independent of S3Guard. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.fs.s3a.tools; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md new file mode 100644 index 0000000000000..e9622ce906d85 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md @@ -0,0 +1,694 @@ + + +# Controlling the S3A Directory Marker Behavior + +## Critical: this is not backwards compatible! + +This document shows how the performance of S3 I/O, especially applications +creating many files (for example Apache Hive) or working with versioned S3 buckets can +increase performance by changing the S3A directory marker retention policy. + +Changing the policy from the default value, `"delete"` _is not backwards compatible_. + +Versions of Hadoop which are incompatible with other marker retention policies, +as of August 2020. + +------------------------------------------------------- +| Branch | Compatible Since | Future Fix Planned? | +|------------|------------------|---------------------| +| Hadoop 2.x | | NO | +| Hadoop 3.0 | | NO | +| Hadoop 3.1 | check | Yes | +| Hadoop 3.2 | check | Yes | +| Hadoop 3.3 | 3.3.1 | Done | +------------------------------------------------------- + +The `s3guard bucket-info` tool [can be used to verify support](#bucket-info). +This allows for a command line check of compatibility, including +in scripts. + +External Hadoop-based applications should also be assumed to be incompatible +unless otherwise stated/known. + +It is only safe change the directory marker policy if the following + conditions are met: + +1. You know exactly which applications are writing to and reading from + (including backing up) an S3 bucket. +2. You know all applications which read data from the bucket are compatible. + +### Applications backing up data. + +It is not enough to have a version of Apache Hadoop which is compatible, any +application which backs up an S3 bucket or copies elsewhere must have an S3 +connector which is compatible. For the Hadoop codebase, that means that if +distcp is used, it _must_ be from a compatible hadoop version. + +### How will incompatible applications/versions fail? + +Applications using an incompatible version of the S3A connector will mistake +directories containing data for empty directories. This means that: + +* Listing directories/directory trees may exclude files which exist. +* Queries across the data will miss data files. +* Renaming a directory to a new location may exclude files underneath. + +The failures are silent: there is no error message, stack trace or +other warning that files may have been missed. They simply aren't +found. + +### If an application has updated a directory tree incompatibly-- what can be done? + +There's a tool on the hadoop command line, [marker tool](#marker-tool) which can audit +a bucket/path for markers, and clean up any markers which were found. +It can be used to make a bucket compatible with older applications. + +Now that this is all clear, let's explain the problem. + + +## Background: Directory Markers: what and why? + +Amazon S3 is not a filesystem, it is an object store. + +The S3A connector not only provides a hadoop-compatible API to interact with +data in S3, it tries to maintain the filesystem metaphor. + +One key aspect of the metaphor of a file system is "directories" + +#### The directory concept + +In normal Unix-style filesystems, the "filesystem" is really a "directory and +file tree" in which files are always stored in "directories" + + +* A directory may contain zero or more files. +* A directory may contain zero or more directories "subdirectories" +* At the base of a filesystem is the "root directory" +* All files MUST be in a directory "the parent directory" +* All directories other than the root directory must be in another directory. +* If a directory contains no files or directories, it is "empty" +* When a directory is _listed_, all files and directories in it are enumerated + and returned to the caller + + +The S3A connector emulates this metaphor by grouping all objects which have +the same prefix as if they are in the same directory tree. + +If there are two objects `a/b/file1` and `a/b/file2` then S3A pretends that there is a +directory `/a/b` containing two files `file1` and `file2`. + +The directory itself does not exist. + +There's a bit of a complication here. + +#### What does `mkdirs()` do? + +1. In HDFS and other "real" filesystems, when `mkdirs()` is invoked on a path +whose parents are all directories, then an _empty directory_ is created. + +1. This directory can be probed for "it exists" and listed (an empty list is +returned) + +1. Files and other directories can be created in it. + + +Lots of code contains a big assumption here: after you create a directory it +exists. They also assume that after files in a directory are deleted, the +directory still exists. + +Given the S3A connector mimics directories just by aggregating objects which share a +prefix, how can you have empty directories? + +The original Hadoop `s3n://` connector created a Directory Marker -any path ending +in `_$folder$` was considered to be a sign that a directory existed. A call to +`mkdir(s3n://bucket/a/b)` would create a new marker object `a/b_$folder$` . + +The S3A also has directory markers, but it just appends a "/" to the directory +name, so `mkdir(s3a://bucket/a/b)` will create a new marker object `a/b/` . + +When a file is created under a path, the directory marker is deleted. And when a +file is deleted, if it was the last file in the directory, the marker is +recreated. + +And, historically, When a path is listed, if a marker to that path is found, *it +has been interpreted as an empty directory.* + +It is that little detail which is the cause of the incompatibility issues. + +## The Problem with Directory Markers + +Creating, deleting and the listing directory markers adds overhead and can slow +down applications. + +Whenever a file is created we have to delete any marker which could exist in +parent directory _or any parent paths_. Rather than do a sequence of probes for +parent markers existing, the connector issues a single request to S3 to delete +all parents. For example, if a file `/a/b/file1` is created, a multi-object +`DELETE` request containing the keys `/a/` and `/a/b/` is issued. +If no markers exists, this is harmless. + +When a file is deleted, a check for the parent directory continuing to exist +(i.e. are there sibling files/directories?), and if not a marker is created. + +This all works well and has worked well for many years. + +However, it turns out to have some scale problems, especially from the delete +call made whenever a file is created. + +1. The number of the objects listed in each request is that of the number of +parent directories: deeper trees create longer requests. + +2. Every single object listed in the delete request is considered to be a write +operation. + +3. In versioned S3 buckets, tombstone markers are added to the S3 indices even +if no object was deleted. + +4. There's also the overhead of actually issuing the request and awaiting the +response. + +Issue #2 has turned out to cause significant problems on some interactions with +large hive tables: + +Because each object listed in a DELETE call is treated as one operation, and +there is (as of summer 2020) a limit of 3500 write requests/second in a directory +tree. +When writing many files to a deep directory tree, it is the delete calls which +create throttling problems. + +The tombstone markers have follow-on consequences -it makes listings against +S3 versioned buckets slower. +This can have adverse effects on those large directories, again. + +## Strategies to avoid marker-related problems. + +### Presto: every path is a directory + +In the Presto [S3 connector](https://prestodb.io/docs/current/connector/hive.html#amazon-s3-configuration), +`mkdirs()` is a no-op. +Whenever it lists any path which isn't an object or a prefix of one more more objects, it returns an +empty listing. That is:; by default, every path is an empty directory. + +Provided no code probes for a directory existing and fails if it is there, this +is very efficient. That's a big requirement however, -one Presto can pull off +because they know how their file uses data in S3. + + +### Hadoop 3.3.1+: marker deletion is now optional + +From Hadoop 3.3.1 onwards, the S3A client can be configured to skip deleting +directory markers when creating files under paths. This removes all scalability +problems caused by deleting these markers -however, it is achieved at the expense +of backwards compatibility. + +## Controlling marker retention with `fs.s3a.directory.marker.retention` + +There is now an option `fs.s3a.directory.marker.retention` which controls how +markers are managed when new files are created + +*Default* `delete`: a request is issued to delete any parental directory markers +whenever a file or directory is created. + +*New* `keep`: No delete request is issued. +Any directory markers which exist are not deleted. +This is *not* backwards compatible + +*New* `authoritative`: directory markers are deleted _except for files created +in "authoritative" directories_. +This is backwards compatible _outside authoritative directories_. + +Until now, the notion of an "authoritative" +directory has only been used as a performance optimization for deployments +where it is known that all Applications are using the same S3Guard metastore +when writing and reading data. +In such a deployment, if it is also known that all applications are using a +compatible version of the s3a connector, then they +can switch to the higher-performance mode for those specific directories. + +Only the default setting, `fs.s3a.directory.marker.retention = delete` is compatible with +every shipping Hadoop releases. + +## Directory Markers and S3Guard + +Applications which interact with S3A in S3A clients with S3Guard enabled still +create and delete markers. There's no attempt to skip operations, such as by having +`mkdirs() `create entries in the DynamoDB table but not the store. +Having the client always update S3 ensures that other applications and clients +do (eventually) see the changes made by the "guarded" application. + +When S3Guard is configured to treat some directories as [Authoritative](s3guard.html#authoritative) +then an S3A connector with a retention policy of `fs.s3a.directory.marker.retention` of +`authoritative` will omit deleting markers in authoritative directories. + +*Note* there may be further changes in directory semantics in "authoritative mode"; +only use in managed applications where all clients are using the same version of +hadoop, and configured consistently. + +## Verifying marker policy with `s3guard bucket-info` + +The `bucket-info` command has been enhanced to support verification from the command +line of bucket policies via the `-marker` option + + +| option | verifies | +|--------|--------| +| `-markers aware` | the hadoop release is "aware" of directory markers | +| `-markers delete` | directory markers are deleted | +| `-markers keep` | directory markers are kept (not backwards compatible) | +| `-markers authoritative` | directory markers are kept in authoritative paths | + +All releases of Hadoop which have been updated to be marker aware will support the `-markers aware` option. + + +1. Updated releases which do not support switching marker retention policy will also support the +`-markers delete` option. + + +Example: `s3guard bucket-info -markers aware` on a compatible release. + +``` +> hadoop s3guard bucket-info -markers aware s3a://landsat-pds/ + Filesystem s3a://landsat-pds + Location: us-west-2 + Filesystem s3a://landsat-pds is not using S3Guard + +... + + Security + Delegation token support is disabled + + The directory marker policy is "delete" + + The S3A connector is compatible with buckets where directory markers are not deleted + Available Policies: delete, keep, authoritative +``` + +The same command will fail on older releases, because the `-markers` option +is unknown + +``` +> hadoop s3guard bucket-info -markers aware s3a://landsat-pds/ +Illegal option -markers +Usage: hadoop bucket-info [OPTIONS] s3a://BUCKET + provide/check S3Guard information about a specific bucket + +Common options: + -guarded - Require S3Guard + -unguarded - Force S3Guard to be disabled + -auth - Require the S3Guard mode to be "authoritative" + -nonauth - Require the S3Guard mode to be "non-authoritative" + -magic - Require the S3 filesystem to be support the "magic" committer + -encryption -require {none, sse-s3, sse-kms} - Require encryption policy + +When possible and not overridden by more specific options, metadata +repository information will be inferred from the S3A URL (if provided) + +Generic options supported are: + -conf - specify an application configuration file + -D - define a value for a given property + +2020-08-12 16:47:16,579 [main] INFO util.ExitUtil (ExitUtil.java:terminate(210)) - Exiting with status 42: Illegal option -markers +```` + +A specific policy check verifies that the connector is configured as desired + +``` +> hadoop s3guard bucket-info -markers delete s3a://landsat-pds/ +Filesystem s3a://landsat-pds +Location: us-west-2 +Filesystem s3a://landsat-pds is not using S3Guard + +... + +The directory marker policy is "delete" +``` + +When probing for a specific policy, the error code "46" is returned if the active policy +does not match that requested: + +``` +> hadoop s3guard bucket-info -markers keep s3a://landsat-pds/ +Filesystem s3a://landsat-pds +Location: us-west-2 +Filesystem s3a://landsat-pds is not using S3Guard + +... + +Security + Delegation token support is disabled + +The directory marker policy is "delete" + +2020-08-12 17:14:30,563 [main] INFO util.ExitUtil (ExitUtil.java:terminate(210)) - Exiting with status 46: 46: Bucket s3a://landsat-pds: required marker policy is "keep" but actual policy is "delete" +``` + + +## The marker tool:`hadoop s3guard markers` + +The marker tool aims to help migration by scanning/auditing directory trees +for surplus markers, and for optionally deleting them. +Leaf-node markers for empty directories are not considered surplus and +will be retained. + +Syntax + +``` +> hadoop s3guard markers -verbose -nonauth +markers (-audit | -clean) [-expected ] [-out ] [-limit ] [-nonauth] [-verbose] + View and manipulate S3 directory markers + +``` + +*Options* + +| Option | Meaning | +|-------------------------|-------------------------| +| `-audit` | Audit the path for surplus markers | +| `-clean` | Clean all surplus markers under a path | +| `-expected ]` | Expected number of markers to find (primarily for testing) | +| `-limit ]` | Limit the number of objects to scan | +| `-nonauth` | Only consider markers in non-authoritative paths as errors | +| `-out ` | Save a list of all markers found to the nominated file | +| `-verbose` | Verbose output | + +*Exit Codes* + +| Code | Meaning | +|-------|---------| +| 0 | Success | +| 3 | interrupted -the value of `-limit` was reached | +| 42 | Usage | +| 46 | Markers were found (see HTTP "406", "unacceptable") | + +All other non-zero status code also indicate errors of some form or other. + +### `markers -audit` + +Audit the path and fail if any markers were found. + + +``` +> hadoop s3guard markers -limit 8000 -audit s3a://landsat-pds/ + +The directory marker policy of s3a://landsat-pds is "Delete" +2020-08-05 13:42:56,079 [main] INFO tools.MarkerTool (DurationInfo.java:(77)) - Starting: marker scan s3a://landsat-pds/ +Scanned 1,000 objects +Scanned 2,000 objects +Scanned 3,000 objects +Scanned 4,000 objects +Scanned 5,000 objects +Scanned 6,000 objects +Scanned 7,000 objects +Scanned 8,000 objects +Limit of scan reached - 8,000 objects +2020-08-05 13:43:01,184 [main] INFO tools.MarkerTool (DurationInfo.java:close(98)) - marker scan s3a://landsat-pds/: duration 0:05.107s +No surplus directory markers were found under s3a://landsat-pds/ +Listing limit reached before completing the scan +2020-08-05 13:43:01,187 [main] INFO util.ExitUtil (ExitUtil.java:terminate(210)) - Exiting with status 3: +``` + +Here the scan reached its object limit before completing the audit; the exit code of 3, "interrupted" indicates this. + +Example: a verbose audit of a bucket whose policy if authoritative -it is not an error if markers +are found under the path `/tables`. + +``` +> bin/hadoop s3guard markers -audit s3a://london/ + + 2020-08-05 18:29:16,473 [main] INFO impl.DirectoryPolicyImpl (DirectoryPolicyImpl.java:getDirectoryPolicy(143)) - Directory markers will be kept on authoritative paths + The directory marker policy of s3a://london is "Authoritative" + Authoritative path list is "/tables" + 2020-08-05 18:29:19,186 [main] INFO tools.MarkerTool (DurationInfo.java:(77)) - Starting: marker scan s3a://london/ + 2020-08-05 18:29:21,610 [main] INFO tools.MarkerTool (DurationInfo.java:close(98)) - marker scan s3a://london/: duration 0:02.425s + Listed 8 objects under s3a://london/ + +Found 3 surplus directory markers under s3a://london/ + s3a://london/tables + s3a://london/tables/tables-4 + s3a://london/tables/tables-4/tables-5 +Found 5 empty directory 'leaf' markers under s3a://london/ + s3a://london/tables/tables-2 + s3a://london/tables/tables-3 + s3a://london/tables/tables-4/tables-5/06 + s3a://london/tables2 + s3a://london/tables3 + These are required to indicate empty directories + Surplus markers were found -failing audit + 2020-08-05 18:29:21,614 [main] INFO util.ExitUtil (ExitUtil.java:terminate(210)) - Exiting with status 46: +``` + +This fails because surplus markers were found. This S3A bucket would *NOT* be safe for older Hadoop versions +to use. + +The `-nonauth` option does not treat markers under authoritative paths as errors: + +``` +bin/hadoop s3guard markers -nonauth -audit s3a://london/ + +2020-08-05 18:31:16,255 [main] INFO impl.DirectoryPolicyImpl (DirectoryPolicyImpl.java:getDirectoryPolicy(143)) - Directory markers will be kept on authoritative paths +The directory marker policy of s3a://london is "Authoritative" +Authoritative path list is "/tables" +2020-08-05 18:31:19,210 [main] INFO tools.MarkerTool (DurationInfo.java:(77)) - Starting: marker scan s3a://london/ +2020-08-05 18:31:22,240 [main] INFO tools.MarkerTool (DurationInfo.java:close(98)) - marker scan s3a://london/: duration 0:03.031s +Listed 8 objects under s3a://london/ + +Found 3 surplus directory markers under s3a://london/ + s3a://london/tables/ + s3a://london/tables/tables-4/ + s3a://london/tables/tables-4/tables-5/ +Found 5 empty directory 'leaf' markers under s3a://london/ + s3a://london/tables/tables-2/ + s3a://london/tables/tables-3/ + s3a://london/tables/tables-4/tables-5/06/ + s3a://london/tables2/ + s3a://london/tables3/ +These are required to indicate empty directories + +Ignoring 3 markers in authoritative paths +``` + +All of this S3A bucket _other_ than the authoritative path `/tables` will be safe for +incompatible Hadoop releases to to use. + + +### `markers clean` + +The `markers clean` command will clean the directory tree of all surplus markers. +The `-verbose` option prints more detail on the operation as well as some IO statistics + +``` +> hadoop s3guard markers -verbose -clean s3a://london/ + +2020-08-05 18:33:25,303 [main] INFO impl.DirectoryPolicyImpl (DirectoryPolicyImpl.java:getDirectoryPolicy(143)) - Directory markers will be kept on authoritative paths +The directory marker policy of s3a://london is "Authoritative" +Authoritative path list is "/tables" +2020-08-05 18:33:28,511 [main] INFO tools.MarkerTool (DurationInfo.java:(77)) - Starting: marker scan s3a://london/ + Directory Marker tables + Directory Marker tables/tables-2/ + Directory Marker tables/tables-3/ + Directory Marker tables/tables-4/ + Directory Marker tables/tables-4/tables-5/ + Directory Marker tables/tables-4/tables-5/06/ + Directory Marker tables2/ + Directory Marker tables3/ +2020-08-05 18:33:31,685 [main] INFO tools.MarkerTool (DurationInfo.java:close(98)) - marker scan s3a://london/: duration 0:03.175s +Listed 8 objects under s3a://london/ + +Found 3 surplus directory markers under s3a://london/ + s3a://london/tables/ + s3a://london/tables/tables-4/ + s3a://london/tables/tables-4/tables-5/ +Found 5 empty directory 'leaf' markers under s3a://london/ + s3a://london/tables/tables-2/ + s3a://london/tables/tables-3/ + s3a://london/tables/tables-4/tables-5/06/ + s3a://london/tables2/ + s3a://london/tables3/ +These are required to indicate empty directories + +3 markers to delete in 1 page of 250 keys/page +2020-08-05 18:33:31,688 [main] INFO tools.MarkerTool (DurationInfo.java:(77)) - Starting: Deleting markers +2020-08-05 18:33:31,812 [main] INFO tools.MarkerTool (DurationInfo.java:close(98)) - Deleting markers: duration 0:00.124s + +Storage Statistics for s3a://london + +op_get_file_status 1 +object_delete_requests 1 +object_list_requests 2 +``` + +The `markers -clean` command _does not_ delete markers above empty directories -only those which have +files underneath. If invoked on a path, it will clean up the directory tree into a state +where it is safe for older versions of Hadoop to interact with. + +Note that if invoked with a `-limit` value, surplus markers found during the scan will be removed, +even though the scan will be considered a failure due to the limit being reached. + +## Advanced Topics + + +### Probing for retention via `PathCapabilities` and `StreamCapabilities` + +An instance of the filesystem can be probed for its directory marker retention ability/ +policy can be probed for through the `org.apache.hadoop.fs.PathCapabilities` interface, +which all FileSystem classes have supported since Hadoop 3.3. + + +| Probe | Meaning | +|-------------------------|-------------------------| +| `fs.s3a.capability.directory.marker.aware` | Does the filesystem support surplus directory markers? | +| `fs.s3a.capability.directory.marker.policy.delete` | Is the bucket policy "delete"? | +| `fs.s3a.capability.directory.marker.policy.keep` | Is the bucket policy "keep"? | +| `fs.s3a.capability.directory.marker.policy.authoritative` | Is the bucket policy "authoritative"? | +| `fs.s3a.capability.directory.marker.action.delete` | If a file was created at this path, would directory markers be deleted? | +| `fs.s3a.capability.directory.marker.action.keep` | If a file was created at this path, would directory markers be retained? | + + +The probe `fs.s3a.capability.directory.marker.aware` allows for a filesystem to be +probed to determine if its file listing policy is "aware" of directory marker retention +-that is: can this s3a client safely work with S3 buckets where markers have not been deleted. + +The `fs.s3a.capability.directory.marker.policy.` probes return the active policy for the bucket. + +The two `fs.s3a.capability.directory.marker.action.` probes dynamically query the marker +retention behavior of a specific path. +That is: if a file was created at that location, would ancestor directory markers +be kept or deleted? + +The `S3AFileSystem` class also implements the `org.apache.hadoop.fs.StreamCapabilities` interface, which +can be used to probe for marker awareness via the `fs.s3a.capability.directory.marker.aware` capability. + +Again, this will be true if-and-only-if the S3A connector is safe to work with S3A buckets/paths where +directories are retained. + +*If an S3A instance, probed by `PathCapabilities` or `StreamCapabilities` for the capability +`fs.s3a.capability.directory.marker.aware` and it returns false, *it is not safe to be used with +S3A paths where markers have been retained*. + +This is programmatic probe -however it can be accessed on the command line via the +external [`cloudstore`](https://github.com/steveloughran/cloudstore) tool: + +``` +> hadoop jar cloudstore-1.0.jar pathcapability fs.s3a.capability.directory.marker.aware s3a://london/ + +Probing s3a://london/ for capability fs.s3a.capability.directory.marker.aware + +Using filesystem s3a://london +Path s3a://london/ has capability fs.s3a.capability.directory.marker.aware +``` + +If the exit code of the command is `0`, then the S3A is safe to work with buckets +where markers have not been deleted. + +The same tool can be used to dynamically probe for the policy. + +Take a bucket with a retention policy of "authoritative" -only paths under `/tables` will have markers retained. + +```xml + + fs.s3a.bucket.london.directory.marker.retention + authoritative + + + fs.s3a.bucket.london.authoritative.path + /tables + ``` +``` + +With this policy the path capability `fs.s3a.capability.directory.marker.action.keep` will hold under +the path `s3a://london/tables` + +``` +bin/hadoop jar cloudstore-1.0.jar pathcapability fs.s3a.capability.directory.marker.action.keep s3a://london/tables +Probing s3a://london/tables for capability fs.s3a.capability.directory.marker.action.keep +2020-08-11 22:03:31,658 [main] INFO impl.DirectoryPolicyImpl (DirectoryPolicyImpl.java:getDirectoryPolicy(143)) + - Directory markers will be kept on authoritative paths +Using filesystem s3a://london +Path s3a://london/tables has capability fs.s3a.capability.directory.marker.action.keep +``` + +However it will not hold for other paths, so indicating that older Hadoop versions will be safe +to work with data written there by this S3A client. + +``` +bin/hadoop jar cloudstore-1.0.jar pathcapability fs.s3a.capability.directory.marker.action.keep s3a://london/tempdir +Probing s3a://london/tempdir for capability fs.s3a.capability.directory.marker.action.keep +2020-08-11 22:06:56,300 [main] INFO impl.DirectoryPolicyImpl (DirectoryPolicyImpl.java:getDirectoryPolicy(143)) + - Directory markers will be kept on authoritative paths +Using filesystem s3a://london +Path s3a://london/tempdir lacks capability fs.s3a.capability.directory.marker.action.keep +2020-08-11 22:06:56,308 [main] INFO util.ExitUtil (ExitUtil.java:terminate(210)) - Exiting with status -1: +``` + + +## Glossary + +#### Directory Marker + +An object in an S3 bucket with a trailing "/", used to indicate that there is a directory at that location. +These are necessary to maintain expectations about directories in an object store: + +1. After `mkdirs(path)`, `exists(path)` holds. +1. After `rm(path/*)`, `exists(path)` holds. + +In previous releases of Hadoop, the marker created by a `mkdirs()` operation was deleted after a file was created. +Rather than make a slow HEAD probe + optional marker DELETE of every parent path element, HADOOP-13164 switched +to enumerating all parent paths and issuing a single bulk DELETE request. +This is faster under light load, but +as each row in the delete consumes one write operation on the allocated IOPs of that bucket partition, creates +load issues when many worker threads/processes are writing to files. +This problem is bad on Apache Hive as: +* The hive partition structure places all files within the same S3 partition. +* As they are deep structures, there are many parent entries to include in the bulk delete calls. +* It's creating a lot temporary files, and still uses rename to commit output. + +Apache Spark has less of an issue when an S3A committer is used -although the partition structure +is the same, the delayed manifestation of output files reduces load. + +#### Leaf Marker + +A directory marker which has not files or directory marker objects underneath. +It genuinely represents an empty directory. + +#### Surplus Marker + +A directory marker which is above one or more files, and so is superfluous. +These are the markers which were traditionally deleted; now it is optional. + +Older versions of Hadoop mistake such surplus markers as Leaf Markers. + +#### Versioned Bucket + +An S3 Bucket which has Object Versioning enabled. + +This provides a backup and recovery mechanism for data within the same +bucket: older objects can be listed and restored through the AWS S3 console +and some applications. + +## References + + + +* [HADOOP-13164](https://issues.apache.org/jira/browse/HADOOP-13164). _Optimize S3AFileSystem::deleteUnnecessaryFakeDirectories._ + +* [HADOOP-13230](https://issues.apache.org/jira/browse/HADOOP-13230). _S3A to optionally retain directory markers_ + +* [HADOOP-16090](https://issues.apache.org/jira/browse/HADOOP-16090). _S3A Client to add explicit support for versioned stores._ + +* [HADOOP-16823](https://issues.apache.org/jira/browse/HADOOP-16823). _Large DeleteObject requests are their own Thundering Herd_ + +* [Object Versioning](https://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html). _Using versioning_ + +* [Optimizing Performance](https://docs.aws.amazon.com/AmazonS3/latest/dev/optimizing-performance.html). _Best Practices Design Patterns: Optimizing Amazon S3 Performance_ diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 964bda49dd069..861da4d82ee23 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -16,18 +16,29 @@ -**NOTE: Hadoop's `s3:` and `s3n:` connectors have been removed. -Please use `s3a:` as the connector to data hosted in S3 with Apache Hadoop.** -**Consult the [s3n documentation](./s3n.html) for migration instructions.** +## Compatibility -See also: + +### Directory Marker Compatibility + +1. This release can safely list/index/read S3 buckets where "empty directory" +markers are retained. + +1. This release can be configured to retain these directory makers at the +expense of being backwards incompatible. + +Consult [Controlling the S3A Directory Marker Behavior](directory_markers.html) for +full details. + +## Documents * [Encryption](./encryption.html) * [Performance](./performance.html) * [S3Guard](./s3guard.html) * [Troubleshooting](./troubleshooting_s3a.html) +* [Controlling the S3A Directory Marker Behavior](directory_markers.html). * [Committing work to S3 with the "S3A Committers"](./committers.html) * [S3A Committers Architecture](./committer_architecture.html) * [Working with IAM Assumed Roles](./assumed_roles.html) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md index 5754f0b5dfdd8..b60d54622ed20 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md @@ -113,7 +113,19 @@ Currently the only Metadata Store-independent setting, besides the implementation class above, are the *allow authoritative* and *fail-on-error* flags. -#### Allow Authoritative +#### Authoritative S3Guard + +Authoritative S3Guard is a complicated configuration which delivers performance +at the expense of being unsafe for other applications to use the same directory +tree/bucket unless configured consistently. + +It can also be used to support [directory marker retention](directory_markers.html) +in higher-performance but non-backwards-compatible modes. + +Most deployments do not use this setting -it is ony used in deployments where +specific parts of a bucket (e.g. Apache Hive managed tables) are known to +have exclusive access by a single application (Hive) and other tools/applications +from exactly the same Hadoop release. The _authoritative_ expression in S3Guard is present in two different layers, for two different reasons: @@ -178,7 +190,7 @@ recommended that you leave the default setting here: false ``` -. + Note that a MetadataStore MAY persist this bit in the directory listings. (Not MUST). diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 5629dab21ff24..e9730444f3a9a 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -324,6 +324,49 @@ Once a bucket is converted to being versioned, it cannot be converted back to being unversioned. +## Testing Different Marker Retention Policy + +Hadoop supports [different policies for directory marker retention](directory_markers.html) +-essentially the classic "delete" and the higher-performance "keep" options; "authoritative" +is just "keep" restricted to a part of the bucket. + +Example: test with `markers=delete` + +``` +mvn verify -Dparallel-tests -DtestsThreadCount=4 -Dmarkers=delete +``` + +Example: test with `markers=keep` + +``` +mvn verify -Dparallel-tests -DtestsThreadCount=4 -Dmarkers=keep +``` + +Example: test with `markers=authoritative` + +``` +mvn verify -Dparallel-tests -DtestsThreadCount=4 -Dmarkers=authoritative +``` + +This final option is of limited use unless paths in the bucket have actually been configured to be +of mixed status; unless anything is set up then the outcome should equal that of "delete" + +### Enabling auditing of markers + +To enable an audit of the output directory of every test suite, +enable the option `fs.s3a.directory.marker.audit` + +``` +-Dfs.s3a.directory.marker.audit=true +``` + +When set, if the marker policy is to delete markers under the test output directory, then +the marker tool audit command will be run. This will fail if a marker was found. + +This adds extra overhead to every operation, but helps verify that the connector is +not keeping markers where it needs to be deleting them -and hence backwards compatibility +is maintained. + ## Scale Tests There are a set of tests designed to measure the scalability and performance diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index d94288dfc307f..a2ee9ea5f7b29 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -25,16 +25,20 @@ import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.contract.s3a.S3AContract; +import org.apache.hadoop.fs.s3a.tools.MarkerTool; import org.apache.hadoop.io.IOUtils; -import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileNotFoundException; import java.io.IOException; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBool; +import static org.apache.hadoop.fs.s3a.S3AUtils.E_FS_CLOSED; +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.UNLIMITED_LISTING; /** * An extension of the contract test base set up for S3A tests. @@ -62,18 +66,43 @@ public void setup() throws Exception { @Override public void teardown() throws Exception { Thread.currentThread().setName("teardown"); + + maybeAuditTestPath(); + super.teardown(); describe("closing file system"); IOUtils.closeStream(getFileSystem()); } - @Before - public void nameThread() { - Thread.currentThread().setName("JUnit-" + getMethodName()); - } - - protected String getMethodName() { - return methodName.getMethodName(); + /** + * Audit the FS under {@link #methodPath()} if + * the test option {@link #DIRECTORY_MARKER_AUDIT} is + * true. + */ + public void maybeAuditTestPath() { + final S3AFileSystem fs = getFileSystem(); + if (fs != null) { + try { + boolean audit = getTestPropertyBool(fs.getConf(), + DIRECTORY_MARKER_AUDIT, false); + Path methodPath = methodPath(); + if (audit + && !fs.getDirectoryMarkerPolicy() + .keepDirectoryMarkers(methodPath) + && fs.isDirectory(methodPath)) { + MarkerTool.ScanResult result = MarkerTool.execMarkerTool(fs, + methodPath, true, 0, UNLIMITED_LISTING, false); + assertEquals("Audit of " + methodPath + " failed: " + result, + 0, result.getExitCode()); + } + } catch (FileNotFoundException ignored) { + } catch (Exception e) { + // If is this is not due to the FS being closed: log. + if (!e.toString().contains(E_FS_CLOSED)) { + LOG.warn("Marker Tool Failure", e); + } + } + } } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java index 0a91102bf5aa6..b1d742a400505 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java @@ -109,7 +109,8 @@ private S3AFileSystem createFullyAuthFS() URI uri = testFS.getUri(); removeBaseAndBucketOverrides(uri.getHost(), config, - METADATASTORE_AUTHORITATIVE); + METADATASTORE_AUTHORITATIVE, + AUTHORITATIVE_PATH); config.setBoolean(METADATASTORE_AUTHORITATIVE, true); final S3AFileSystem newFS = createFS(uri, config); // set back the same metadata store instance @@ -124,7 +125,8 @@ private S3AFileSystem createSinglePathAuthFS(String authPath) URI uri = testFS.getUri(); removeBaseAndBucketOverrides(uri.getHost(), config, - METADATASTORE_AUTHORITATIVE); + METADATASTORE_AUTHORITATIVE, + AUTHORITATIVE_PATH); config.set(AUTHORITATIVE_PATH, authPath.toString()); final S3AFileSystem newFS = createFS(uri, config); // set back the same metadata store instance @@ -139,7 +141,8 @@ private S3AFileSystem createMultiPathAuthFS(String first, String middle, String URI uri = testFS.getUri(); removeBaseAndBucketOverrides(uri.getHost(), config, - METADATASTORE_AUTHORITATIVE); + METADATASTORE_AUTHORITATIVE, + AUTHORITATIVE_PATH); config.set(AUTHORITATIVE_PATH, first + "," + middle + "," + last); final S3AFileSystem newFS = createFS(uri, config); // set back the same metadata store instance @@ -155,7 +158,8 @@ private S3AFileSystem createRawFS() throws Exception { removeBaseAndBucketOverrides(uri.getHost(), config, S3_METADATA_STORE_IMPL); removeBaseAndBucketOverrides(uri.getHost(), config, - METADATASTORE_AUTHORITATIVE); + METADATASTORE_AUTHORITATIVE, + AUTHORITATIVE_PATH); return createFS(uri, config); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java index 6be9003e4ec38..8c215d79ea680 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java @@ -75,7 +75,11 @@ public void testNoBucketProbing() throws Exception { // the exception must not be caught and marked down to an FNFE expectUnknownStore(() -> fs.exists(src)); - expectUnknownStore(() -> fs.isFile(src)); + // now that isFile() only does a HEAD, it will get a 404 without + // the no-such-bucket error. + assertFalse("isFile(" + src + ")" + + " was expected to complete by returning false", + fs.isFile(src)); expectUnknownStore(() -> fs.isDirectory(src)); expectUnknownStore(() -> fs.mkdirs(src)); expectUnknownStore(() -> fs.delete(src)); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java index f086a08201cd7..1c395b2adcfc4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java @@ -20,8 +20,13 @@ import java.io.IOException; import java.nio.file.AccessDeniedException; +import java.util.Arrays; +import java.util.Collection; +import org.assertj.core.api.Assertions; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -31,13 +36,26 @@ import org.apache.hadoop.io.IOUtils; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP; +import static org.apache.hadoop.fs.s3a.Constants.ETAG_CHECKSUM_ENABLED; +import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Concrete class that extends {@link AbstractTestS3AEncryption} * and tests SSE-C encryption. + * HEAD requests against SSE-C-encrypted data will fail if the wrong key + * is presented, so the tests are very brittle to S3Guard being on vs. off. + * Equally "vexing" has been the optimizations of getFileStatus(), wherein + * LIST comes before HEAD path + / */ +@RunWith(Parameterized.class) public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { private static final String SERVICE_AMAZON_S3_STATUS_CODE_403 @@ -52,18 +70,67 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { = "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8="; private static final int TEST_FILE_LEN = 2048; + /** + * Parameterization. + */ + @Parameterized.Parameters(name = "{0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {"raw-keep-markers", false, true}, + {"raw-delete-markers", false, false}, + {"guarded-keep-markers", true, true}, + {"guarded-delete-markers", true, false} + }); + } + + /** + * Parameter: should the stores be guarded? + */ + private final boolean s3guard; + + /** + * Parameter: should directory markers be retained? + */ + private final boolean keepMarkers; + /** * Filesystem created with a different key. */ - private FileSystem fsKeyB; + private S3AFileSystem fsKeyB; + + public ITestS3AEncryptionSSEC(final String name, + final boolean s3guard, + final boolean keepMarkers) { + this.s3guard = s3guard; + this.keepMarkers = keepMarkers; + } @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); disableFilesystemCaching(conf); - conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM, + String bucketName = getTestBucketName(conf); + removeBucketOverrides(bucketName, conf, + S3_METADATA_STORE_IMPL); + if (!s3guard) { + // in a raw run remove all s3guard settings + removeBaseAndBucketOverrides(bucketName, conf, + S3_METADATA_STORE_IMPL); + } + // directory marker options + removeBaseAndBucketOverrides(bucketName, conf, + DIRECTORY_MARKER_POLICY, + ETAG_CHECKSUM_ENABLED, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY); + conf.set(DIRECTORY_MARKER_POLICY, + keepMarkers + ? DIRECTORY_MARKER_POLICY_KEEP + : DIRECTORY_MARKER_POLICY_DELETE); + conf.set(SERVER_SIDE_ENCRYPTION_ALGORITHM, getSSEAlgorithm().getMethod()); - conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, KEY_1); + conf.set(SERVER_SIDE_ENCRYPTION_KEY, KEY_1); + conf.setBoolean(ETAG_CHECKSUM_ENABLED, true); return conf; } @@ -109,31 +176,19 @@ public void testCreateFileAndReadWithDifferentEncryptionKey() throws } /** - * While each object has its own key and should be distinct, this verifies - * that hadoop treats object keys as a filesystem path. So if a top level - * dir is encrypted with keyA, a sublevel dir cannot be accessed with a - * different keyB. - * - * This is expected AWS S3 SSE-C behavior. * + * You can use a different key under a sub directory, even if you + * do not have permissions to read the marker. * @throws Exception */ @Test public void testCreateSubdirWithDifferentKey() throws Exception { - requireUnguardedFilesystem(); - - intercept(AccessDeniedException.class, - SERVICE_AMAZON_S3_STATUS_CODE_403, - () -> { - Path base = path("testCreateSubdirWithDifferentKey"); - Path nestedDirectory = new Path(base, "nestedDir"); - fsKeyB = createNewFileSystemWithSSECKey( - KEY_2); - getFileSystem().mkdirs(base); - fsKeyB.mkdirs(nestedDirectory); - // expected to fail - return fsKeyB.getFileStatus(nestedDirectory); - }); + Path base = path("testCreateSubdirWithDifferentKey"); + Path nestedDirectory = new Path(base, "nestedDir"); + fsKeyB = createNewFileSystemWithSSECKey( + KEY_2); + getFileSystem().mkdirs(base); + fsKeyB.mkdirs(nestedDirectory); } /** @@ -176,14 +231,11 @@ public void testRenameFile() throws Exception { } /** - * It is possible to list the contents of a directory up to the actual - * end of the nested directories. This is due to how S3A mocks the - * directories and how prefixes work in S3. + * Directory listings always work. * @throws Exception */ @Test public void testListEncryptedDir() throws Exception { - requireUnguardedFilesystem(); Path pathABC = path("testListEncryptedDir/a/b/c/"); Path pathAB = pathABC.getParent(); @@ -196,17 +248,11 @@ public void testListEncryptedDir() throws Exception { fsKeyB.listFiles(pathA, true); fsKeyB.listFiles(pathAB, true); - - //Until this point, no exception is thrown about access - intercept(AccessDeniedException.class, - SERVICE_AMAZON_S3_STATUS_CODE_403, - () -> { - fsKeyB.listFiles(pathABC, false); - }); + fsKeyB.listFiles(pathABC, false); Configuration conf = this.createConfiguration(); - conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM); - conf.unset(Constants.SERVER_SIDE_ENCRYPTION_KEY); + conf.unset(SERVER_SIDE_ENCRYPTION_ALGORITHM); + conf.unset(SERVER_SIDE_ENCRYPTION_KEY); S3AContract contract = (S3AContract) createContract(conf); contract.init(); @@ -215,20 +261,14 @@ public void testListEncryptedDir() throws Exception { //unencrypted can access until the final directory unencryptedFileSystem.listFiles(pathA, true); unencryptedFileSystem.listFiles(pathAB, true); - AWSBadRequestException ex = intercept(AWSBadRequestException.class, - () -> { - unencryptedFileSystem.listFiles(pathABC, false); - }); + unencryptedFileSystem.listFiles(pathABC, false); } /** - * Much like the above list encrypted directory test, you cannot get the - * metadata of an object without the correct encryption key. - * @throws Exception + * listStatus also works with encrypted directories and key mismatch. */ @Test public void testListStatusEncryptedDir() throws Exception { - requireUnguardedFilesystem(); Path pathABC = path("testListStatusEncryptedDir/a/b/c/"); Path pathAB = pathABC.getParent(); @@ -240,17 +280,14 @@ public void testListStatusEncryptedDir() throws Exception { fsKeyB.listStatus(pathA); fsKeyB.listStatus(pathAB); - //Until this point, no exception is thrown about access - intercept(AccessDeniedException.class, - SERVICE_AMAZON_S3_STATUS_CODE_403, - () -> { - fsKeyB.listStatus(pathABC); - }); + // this used to raise 403, but with LIST before HEAD, + // no longer true. + fsKeyB.listStatus(pathABC); //Now try it with an unencrypted filesystem. Configuration conf = createConfiguration(); - conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM); - conf.unset(Constants.SERVER_SIDE_ENCRYPTION_KEY); + conf.unset(SERVER_SIDE_ENCRYPTION_ALGORITHM); + conf.unset(SERVER_SIDE_ENCRYPTION_KEY); S3AContract contract = (S3AContract) createContract(conf); contract.init(); @@ -259,21 +296,15 @@ public void testListStatusEncryptedDir() throws Exception { //unencrypted can access until the final directory unencryptedFileSystem.listStatus(pathA); unencryptedFileSystem.listStatus(pathAB); - - intercept(AWSBadRequestException.class, - () -> { - unencryptedFileSystem.listStatus(pathABC); - }); + unencryptedFileSystem.listStatus(pathABC); } /** - * Much like trying to access a encrypted directory, an encrypted file cannot - * have its metadata read, since both are technically an object. + * An encrypted file cannot have its metadata read. * @throws Exception */ @Test public void testListStatusEncryptedFile() throws Exception { - requireUnguardedFilesystem(); Path pathABC = path("testListStatusEncryptedFile/a/b/c/"); assertTrue("mkdirs failed", getFileSystem().mkdirs(pathABC)); @@ -283,23 +314,15 @@ public void testListStatusEncryptedFile() throws Exception { fsKeyB = createNewFileSystemWithSSECKey(KEY_4); //Until this point, no exception is thrown about access - intercept(AccessDeniedException.class, - SERVICE_AMAZON_S3_STATUS_CODE_403, - () -> { - fsKeyB.listStatus(fileToStat); - }); + if (!fsKeyB.hasMetadataStore()) { + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> fsKeyB.listStatus(fileToStat)); + } else { + fsKeyB.listStatus(fileToStat); + } } - /** - * Skip the test case if S3Guard is enabled; generally this is because - * list and GetFileStatus calls can succeed even with different keys. - */ - protected void requireUnguardedFilesystem() { - assume("Filesystem has a metastore", - !getFileSystem().hasMetadataStore()); - } - - /** * It is possible to delete directories without the proper encryption key and * the hierarchy above it. @@ -308,7 +331,7 @@ protected void requireUnguardedFilesystem() { */ @Test public void testDeleteEncryptedObjectWithDifferentKey() throws Exception { - requireUnguardedFilesystem(); + //requireUnguardedFilesystem(); Path pathABC = path("testDeleteEncryptedObjectWithDifferentKey/a/b/c/"); Path pathAB = pathABC.getParent(); @@ -317,12 +340,13 @@ public void testDeleteEncryptedObjectWithDifferentKey() throws Exception { Path fileToDelete = new Path(pathABC, "filetobedeleted.txt"); writeThenReadFile(fileToDelete, TEST_FILE_LEN); fsKeyB = createNewFileSystemWithSSECKey(KEY_4); - intercept(AccessDeniedException.class, - SERVICE_AMAZON_S3_STATUS_CODE_403, - () -> { - fsKeyB.delete(fileToDelete, false); - }); - + if (!fsKeyB.hasMetadataStore()) { + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> fsKeyB.delete(fileToDelete, false)); + } else { + fsKeyB.delete(fileToDelete, false); + } //This is possible fsKeyB.delete(pathABC, true); fsKeyB.delete(pathAB, true); @@ -330,15 +354,33 @@ public void testDeleteEncryptedObjectWithDifferentKey() throws Exception { assertPathDoesNotExist("expected recursive delete", fileToDelete); } - private FileSystem createNewFileSystemWithSSECKey(String sseCKey) throws + /** + * getFileChecksum always goes to S3, so when + * the caller lacks permissions, it fails irrespective + * of guard. + */ + @Test + public void testChecksumRequiresReadAccess() throws Throwable { + Path path = path("tagged-file"); + S3AFileSystem fs = getFileSystem(); + touch(fs, path); + Assertions.assertThat(fs.getFileChecksum(path)) + .isNotNull(); + fsKeyB = createNewFileSystemWithSSECKey(KEY_4); + intercept(AccessDeniedException.class, + SERVICE_AMAZON_S3_STATUS_CODE_403, + () -> fsKeyB.getFileChecksum(path)); + } + + private S3AFileSystem createNewFileSystemWithSSECKey(String sseCKey) throws IOException { Configuration conf = this.createConfiguration(); - conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, sseCKey); + conf.set(SERVER_SIDE_ENCRYPTION_KEY, sseCKey); S3AContract contract = (S3AContract) createContract(conf); contract.init(); FileSystem fileSystem = contract.getTestFileSystem(); - return fileSystem; + return (S3AFileSystem) fileSystem; } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index e54fd97a6af1e..46e6f5fcea74f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -18,15 +18,14 @@ package org.apache.hadoop.fs.s3a; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; + +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; +import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest; + -import org.assertj.core.api.Assertions; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -39,26 +38,21 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; -import java.util.UUID; -import java.util.concurrent.Callable; + import static org.apache.hadoop.fs.contract.ContractTestUtils.*; -import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.test.GenericTestUtils.getTestDir; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** - * Use metrics to assert about the cost of file status queries. - * {@link S3AFileSystem#getFileStatus(Path)}. - * Parameterized on guarded vs raw. + * Use metrics to assert about the cost of file API calls. + * Parameterized on guarded vs raw. and directory marker keep vs delete */ @RunWith(Parameterized.class) -public class ITestS3AFileOperationCost extends AbstractS3ATestBase { - - private MetricDiff metadataRequests; - private MetricDiff listRequests; +public class ITestS3AFileOperationCost extends AbstractS3ACostTest { private static final Logger LOG = LoggerFactory.getLogger(ITestS3AFileOperationCost.class); @@ -69,103 +63,62 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase { @Parameterized.Parameters(name = "{0}") public static Collection params() { return Arrays.asList(new Object[][]{ - {"raw", false}, - {"guarded", true} + {"raw-keep-markers", false, true, false}, + {"raw-delete-markers", false, false, false}, + {"nonauth-keep-markers", true, true, false}, + {"auth-delete-markers", true, false, true} }); } - private final String name; - - private final boolean s3guard; - - public ITestS3AFileOperationCost(final String name, final boolean s3guard) { - this.name = name; - this.s3guard = s3guard; - } - - @Override - public Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - String bucketName = getTestBucketName(conf); - removeBucketOverrides(bucketName, conf, - S3_METADATA_STORE_IMPL); - if (!s3guard) { - // in a raw run remove all s3guard settings - removeBaseAndBucketOverrides(bucketName, conf, - S3_METADATA_STORE_IMPL); - } - disableFilesystemCaching(conf); - return conf; - } - @Override - public void setup() throws Exception { - super.setup(); - if (s3guard) { - // s3guard is required for those test runs where any of the - // guard options are set - assumeS3GuardState(true, getConfiguration()); - } - S3AFileSystem fs = getFileSystem(); - metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS); - listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS); - skipDuringFaultInjection(fs); + public ITestS3AFileOperationCost(final String name, + final boolean s3guard, + final boolean keepMarkers, + final boolean authoritative) { + super(s3guard, keepMarkers, authoritative); } + /** + * Test the cost of {@code listLocatedStatus(file)}. + * There's a minor inefficiency in that calling this on + * a file in S3Guard still executes a LIST call, even + * though the file record is in the store. + */ @Test public void testCostOfLocatedFileStatusOnFile() throws Throwable { describe("performing listLocatedStatus on a file"); - Path file = path(getMethodName() + ".txt"); + Path file = file(methodPath()); S3AFileSystem fs = getFileSystem(); - touch(fs, file); - resetMetricDiffs(); - fs.listLocatedStatus(file); - if (!fs.hasMetadataStore()) { - // Unguarded FS. - metadataRequests.assertDiffEquals(1); - } - listRequests.assertDiffEquals(1); + verifyMetrics(() -> fs.listLocatedStatus(file), + whenRaw(FILE_STATUS_FILE_PROBE + .plus(LIST_LOCATED_STATUS_LIST_OP)), + whenAuthoritative(LIST_LOCATED_STATUS_LIST_OP), + whenNonauth(LIST_LOCATED_STATUS_LIST_OP)); } @Test public void testCostOfListLocatedStatusOnEmptyDir() throws Throwable { describe("performing listLocatedStatus on an empty dir"); - Path dir = path(getMethodName()); + Path dir = dir(methodPath()); S3AFileSystem fs = getFileSystem(); - fs.mkdirs(dir); - resetMetricDiffs(); - fs.listLocatedStatus(dir); - if (!fs.hasMetadataStore()) { - // Unguarded FS. - verifyOperationCount(2, 1); - } else { - if (fs.allowAuthoritative(dir)) { - verifyOperationCount(0, 0); - } else { - verifyOperationCount(0, 1); - } - } + verifyMetrics(() -> + fs.listLocatedStatus(dir), + whenRaw(LIST_LOCATED_STATUS_LIST_OP + .plus(GET_FILE_STATUS_ON_EMPTY_DIR)), + whenAuthoritative(NO_IO), + whenNonauth(LIST_LOCATED_STATUS_LIST_OP)); } @Test public void testCostOfListLocatedStatusOnNonEmptyDir() throws Throwable { describe("performing listLocatedStatus on a non empty dir"); - Path dir = path(getMethodName() + "dir"); + Path dir = dir(methodPath()); S3AFileSystem fs = getFileSystem(); - fs.mkdirs(dir); - Path file = new Path(dir, "file.txt"); - touch(fs, file); - resetMetricDiffs(); - fs.listLocatedStatus(dir); - if (!fs.hasMetadataStore()) { - // Unguarded FS. - verifyOperationCount(0, 1); - } else { - if(fs.allowAuthoritative(dir)) { - verifyOperationCount(0, 0); - } else { - verifyOperationCount(0, 1); - } - } + Path file = file(new Path(dir, "file.txt")); + verifyMetrics(() -> + fs.listLocatedStatus(dir), + whenRaw(LIST_LOCATED_STATUS_LIST_OP), + whenAuthoritative(NO_IO), + whenNonauth(LIST_LOCATED_STATUS_LIST_OP)); } @Test @@ -174,36 +127,27 @@ public void testCostOfListFilesOnFile() throws Throwable { Path file = path(getMethodName() + ".txt"); S3AFileSystem fs = getFileSystem(); touch(fs, file); - resetMetricDiffs(); - fs.listFiles(file, true); - if (!fs.hasMetadataStore()) { - metadataRequests.assertDiffEquals(1); - } else { - if (fs.allowAuthoritative(file)) { - listRequests.assertDiffEquals(0); - } else { - listRequests.assertDiffEquals(1); - } - } + verifyMetrics(() -> + fs.listFiles(file, true), + whenRaw(LIST_LOCATED_STATUS_LIST_OP + .plus(GET_FILE_STATUS_ON_FILE)), + whenAuthoritative(NO_IO), + whenNonauth(LIST_LOCATED_STATUS_LIST_OP)); } @Test public void testCostOfListFilesOnEmptyDir() throws Throwable { - describe("Performing listFiles() on an empty dir"); + describe("Perpforming listFiles() on an empty dir with marker"); + // this attem Path dir = path(getMethodName()); S3AFileSystem fs = getFileSystem(); fs.mkdirs(dir); - resetMetricDiffs(); - fs.listFiles(dir, true); - if (!fs.hasMetadataStore()) { - verifyOperationCount(2, 1); - } else { - if (fs.allowAuthoritative(dir)) { - verifyOperationCount(0, 0); - } else { - verifyOperationCount(0, 1); - } - } + verifyMetrics(() -> + fs.listFiles(dir, true), + whenRaw(LIST_FILES_LIST_OP + .plus(GET_FILE_STATUS_ON_EMPTY_DIR)), + whenAuthoritative(NO_IO), + whenNonauth(LIST_FILES_LIST_OP)); } @Test @@ -214,17 +158,11 @@ public void testCostOfListFilesOnNonEmptyDir() throws Throwable { fs.mkdirs(dir); Path file = new Path(dir, "file.txt"); touch(fs, file); - resetMetricDiffs(); - fs.listFiles(dir, true); - if (!fs.hasMetadataStore()) { - verifyOperationCount(0, 1); - } else { - if (fs.allowAuthoritative(dir)) { - verifyOperationCount(0, 0); - } else { - verifyOperationCount(0, 1); - } - } + verifyMetrics(() -> + fs.listFiles(dir, true), + whenRaw(LIST_FILES_LIST_OP), + whenAuthoritative(NO_IO), + whenNonauth(LIST_FILES_LIST_OP)); } @Test @@ -232,118 +170,70 @@ public void testCostOfListFilesOnNonExistingDir() throws Throwable { describe("Performing listFiles() on a non existing dir"); Path dir = path(getMethodName()); S3AFileSystem fs = getFileSystem(); - resetMetricDiffs(); - intercept(FileNotFoundException.class, - () -> fs.listFiles(dir, true)); - verifyOperationCount(2, 2); + verifyMetricsIntercepting(FileNotFoundException.class, "", + () -> fs.listFiles(dir, true), + whenRaw(LIST_FILES_LIST_OP + .plus(GET_FILE_STATUS_FNFE))); } @Test public void testCostOfGetFileStatusOnFile() throws Throwable { describe("performing getFileStatus on a file"); - Path simpleFile = path("simple.txt"); - S3AFileSystem fs = getFileSystem(); - touch(fs, simpleFile); - resetMetricDiffs(); - FileStatus status = fs.getFileStatus(simpleFile); + Path simpleFile = file(methodPath()); + S3AFileStatus status = verifyRawInnerGetFileStatus(simpleFile, true, + StatusProbeEnum.ALL, + GET_FILE_STATUS_ON_FILE); assertTrue("not a file: " + status, status.isFile()); - if (!fs.hasMetadataStore()) { - metadataRequests.assertDiffEquals(1); - } - listRequests.assertDiffEquals(0); - } - - private void resetMetricDiffs() { - reset(metadataRequests, listRequests); - } - - /** - * Verify that the head and list calls match expectations, - * then reset the counters ready for the next operation. - * @param head expected HEAD count - * @param list expected LIST count - */ - private void verifyOperationCount(int head, int list) { - metadataRequests.assertDiffEquals(head); - listRequests.assertDiffEquals(list); - metadataRequests.reset(); - listRequests.reset(); } @Test public void testCostOfGetFileStatusOnEmptyDir() throws Throwable { describe("performing getFileStatus on an empty directory"); - S3AFileSystem fs = getFileSystem(); - Path dir = path("empty"); - fs.mkdirs(dir); - resetMetricDiffs(); - S3AFileStatus status = fs.innerGetFileStatus(dir, true, - StatusProbeEnum.ALL); + Path dir = dir(methodPath()); + S3AFileStatus status = verifyRawInnerGetFileStatus(dir, true, + StatusProbeEnum.ALL, + GET_FILE_STATUS_ON_DIR_MARKER); assertSame("not empty: " + status, Tristate.TRUE, status.isEmptyDirectory()); - - if (!fs.hasMetadataStore()) { - metadataRequests.assertDiffEquals(2); - } - listRequests.assertDiffEquals(0); - // but now only ask for the directories and the file check is skipped. - resetMetricDiffs(); - fs.innerGetFileStatus(dir, false, - StatusProbeEnum.DIRECTORIES); - if (!fs.hasMetadataStore()) { - metadataRequests.assertDiffEquals(1); - } + verifyRawInnerGetFileStatus(dir, false, + StatusProbeEnum.DIRECTORIES, + FILE_STATUS_DIR_PROBE); + + // now look at isFile/isDir against the same entry + isDir(dir, true, FILE_STATUS_DIR_PROBE); + isFile(dir, false, FILE_STATUS_FILE_PROBE); } @Test public void testCostOfGetFileStatusOnMissingFile() throws Throwable { describe("performing getFileStatus on a missing file"); - S3AFileSystem fs = getFileSystem(); - Path path = path("missing"); - resetMetricDiffs(); - intercept(FileNotFoundException.class, - () -> fs.getFileStatus(path)); - metadataRequests.assertDiffEquals(2); - listRequests.assertDiffEquals(1); + interceptRawGetFileStatusFNFE(methodPath(), false, + StatusProbeEnum.ALL, + GET_FILE_STATUS_FNFE); } @Test - public void testCostOfGetFileStatusOnMissingSubPath() throws Throwable { - describe("performing getFileStatus on a missing file"); - S3AFileSystem fs = getFileSystem(); - Path path = path("missingdir/missingpath"); - resetMetricDiffs(); - intercept(FileNotFoundException.class, - () -> fs.getFileStatus(path)); - metadataRequests.assertDiffEquals(2); - listRequests.assertDiffEquals(1); + public void testIsDirIsFileMissingPath() throws Throwable { + describe("performing isDir and isFile on a missing file"); + Path path = methodPath(); + // now look at isFile/isDir against the same entry + isDir(path, false, + FILE_STATUS_DIR_PROBE); + isFile(path, false, + FILE_STATUS_FILE_PROBE); } @Test public void testCostOfGetFileStatusOnNonEmptyDir() throws Throwable { describe("performing getFileStatus on a non-empty directory"); - S3AFileSystem fs = getFileSystem(); - Path dir = path("empty"); - fs.mkdirs(dir); - Path simpleFile = new Path(dir, "simple.txt"); - touch(fs, simpleFile); - resetMetricDiffs(); - S3AFileStatus status = fs.innerGetFileStatus(dir, true, - StatusProbeEnum.ALL); - if (status.isEmptyDirectory() == Tristate.TRUE) { - // erroneous state - String fsState = fs.toString(); - fail("FileStatus says directory isempty: " + status - + "\n" + ContractTestUtils.ls(fs, dir) - + "\n" + fsState); - } - if (!fs.hasMetadataStore()) { - metadataRequests.assertDiffEquals(2); - listRequests.assertDiffEquals(1); - } + Path dir = dir(methodPath()); + file(new Path(dir, "simple.txt")); + S3AFileStatus status = verifyRawInnerGetFileStatus(dir, true, + StatusProbeEnum.ALL, + GET_FILE_STATUS_ON_DIR); + assertEmptyDirStatus(status, Tristate.FALSE); } - @Test public void testCostOfCopyFromLocalFile() throws Throwable { describe("testCostOfCopyFromLocalFile"); @@ -361,19 +251,18 @@ public void testCostOfCopyFromLocalFile() throws Throwable { byte[] data = dataset(len, 'A', 'Z'); writeDataset(localFS, localPath, data, len, 1024, true); S3AFileSystem s3a = getFileSystem(); - MetricDiff copyLocalOps = new MetricDiff(s3a, - INVOCATION_COPY_FROM_LOCAL_FILE); - MetricDiff putRequests = new MetricDiff(s3a, - OBJECT_PUT_REQUESTS); - MetricDiff putBytes = new MetricDiff(s3a, - OBJECT_PUT_BYTES); - - Path remotePath = path("copied"); - s3a.copyFromLocalFile(false, true, localPath, remotePath); + + + Path remotePath = methodPath(); + + verifyMetrics(() -> { + s3a.copyFromLocalFile(false, true, localPath, remotePath); + return "copy"; + }, + with(INVOCATION_COPY_FROM_LOCAL_FILE, 1), + with(OBJECT_PUT_REQUESTS, 1), + with(OBJECT_PUT_BYTES, len)); verifyFileContents(s3a, remotePath, data); - copyLocalOps.assertDiffEquals(1); - putRequests.assertDiffEquals(1); - putBytes.assertDiffEquals(len); // print final stats LOG.info("Filesystem {}", s3a); } finally { @@ -381,268 +270,123 @@ public void testCostOfCopyFromLocalFile() throws Throwable { } } - private boolean reset(MetricDiff... diffs) { - for (MetricDiff diff : diffs) { - diff.reset(); - } - return true; - } - - @Test - public void testFakeDirectoryDeletion() throws Throwable { - describe("Verify whether create file works after renaming a file. " - + "In S3, rename deletes any fake directories as a part of " - + "clean up activity"); - S3AFileSystem fs = getFileSystem(); - - Path srcBaseDir = path("src"); - mkdirs(srcBaseDir); - MetricDiff deleteRequests = - new MetricDiff(fs, Statistic.OBJECT_DELETE_REQUESTS); - MetricDiff directoriesDeleted = - new MetricDiff(fs, Statistic.DIRECTORIES_DELETED); - MetricDiff fakeDirectoriesDeleted = - new MetricDiff(fs, Statistic.FAKE_DIRECTORIES_DELETED); - MetricDiff directoriesCreated = - new MetricDiff(fs, Statistic.DIRECTORIES_CREATED); - - // when you call toString() on this, you get the stats - // so it gets auto-evaluated in log calls. - Object summary = new Object() { - @Override - public String toString() { - return String.format("[%s, %s, %s, %s]", - directoriesCreated, directoriesDeleted, - deleteRequests, fakeDirectoriesDeleted); - } - }; - - // reset operation to invoke - Callable reset = () -> - reset(deleteRequests, directoriesCreated, directoriesDeleted, - fakeDirectoriesDeleted); - - Path srcDir = new Path(srcBaseDir, "1/2/3/4/5/6"); - int srcDirDepth = directoriesInPath(srcDir); - // one dir created, one removed - mkdirs(srcDir); - String state = "after mkdir(srcDir) " + summary; - directoriesCreated.assertDiffEquals(state, 1); - deleteRequests.assertDiffEquals(state, 1); - directoriesDeleted.assertDiffEquals(state, 0); - // HADOOP-14255 deletes unnecessary fake directory objects in mkdirs() - fakeDirectoriesDeleted.assertDiffEquals(state, srcDirDepth - 1); - reset.call(); - - // creating a file should trigger demise of the src dir - final Path srcFilePath = new Path(srcDir, "source.txt"); - touch(fs, srcFilePath); - state = "after touch(fs, srcFilePath) " + summary; - deleteRequests.assertDiffEquals(state, 1); - directoriesCreated.assertDiffEquals(state, 0); - directoriesDeleted.assertDiffEquals(state, 0); - fakeDirectoriesDeleted.assertDiffEquals(state, srcDirDepth); - - reset.call(); - - // create a directory tree, expect the dir to be created and - // a request to delete all parent directories made. - Path destBaseDir = path("dest"); - Path destDir = new Path(destBaseDir, "1/2/3/4/5/6"); - Path destFilePath = new Path(destDir, "dest.txt"); - mkdirs(destDir); - state = "after mkdir(destDir) " + summary; - - int destDirDepth = directoriesInPath(destDir); - directoriesCreated.assertDiffEquals(state, 1); - deleteRequests.assertDiffEquals(state, 1); - directoriesDeleted.assertDiffEquals(state, 0); - fakeDirectoriesDeleted.assertDiffEquals(state, destDirDepth - 1); - - // create a new source file. - // Explicitly use a new path object to guarantee that the parent paths - // are different object instances - final Path srcFile2 = new Path(srcDir.toUri() + "/source2.txt"); - touch(fs, srcFile2); - - reset.call(); - - // rename the source file to the destination file. - // this tests the file rename path, not the dir rename path - // as srcFile2 exists, the parent dir of srcFilePath must not be created. - fs.rename(srcFilePath, destFilePath); - state = String.format("after rename(srcFilePath, destFilePath)" - + " %s dest dir depth=%d", - summary, - destDirDepth); - - directoriesCreated.assertDiffEquals(state, 0); - // one for the renamed file, one for the parent of the dest dir - deleteRequests.assertDiffEquals(state, 2); - directoriesDeleted.assertDiffEquals(state, 0); - fakeDirectoriesDeleted.assertDiffEquals(state, destDirDepth); - - // these asserts come after the checks on iop counts, so they don't - // interfere - assertIsFile(destFilePath); - assertIsDirectory(srcDir); - assertPathDoesNotExist("should have gone in the rename", srcFilePath); - reset.call(); - - // rename the source file2 to the (no longer existing - // this tests the file rename path, not the dir rename path - // as srcFile2 exists, the parent dir of srcFilePath must not be created. - fs.rename(srcFile2, srcFilePath); - state = String.format("after rename(%s, %s) %s dest dir depth=%d", - srcFile2, srcFilePath, - summary, - destDirDepth); - - // here we expect there to be no fake directories - directoriesCreated.assertDiffEquals(state, 0); - // one for the renamed file only - deleteRequests.assertDiffEquals(state, 1); - directoriesDeleted.assertDiffEquals(state, 0); - fakeDirectoriesDeleted.assertDiffEquals(state, 0); - } - - private int directoriesInPath(Path path) { - return path.isRoot() ? 0 : 1 + directoriesInPath(path.getParent()); - } - @Test - public void testCostOfRootRename() throws Throwable { - describe("assert that a root directory rename doesn't" - + " do much in terms of parent dir operations"); + public void testDirProbes() throws Throwable { + describe("Test directory probe cost"); + assumeUnguarded(); S3AFileSystem fs = getFileSystem(); + // Create the empty directory. + Path emptydir = dir(methodPath()); - // unique name, so that even when run in parallel tests, there's no conflict - String uuid = UUID.randomUUID().toString(); - Path src = new Path("/src-" + uuid); - Path dest = new Path("/dest-" + uuid); + // head probe fails + interceptRawGetFileStatusFNFE(emptydir, false, + StatusProbeEnum.HEAD_ONLY, + FILE_STATUS_FILE_PROBE); - try { - MetricDiff deleteRequests = - new MetricDiff(fs, Statistic.OBJECT_DELETE_REQUESTS); - MetricDiff directoriesDeleted = - new MetricDiff(fs, Statistic.DIRECTORIES_DELETED); - MetricDiff fakeDirectoriesDeleted = - new MetricDiff(fs, Statistic.FAKE_DIRECTORIES_DELETED); - MetricDiff directoriesCreated = - new MetricDiff(fs, Statistic.DIRECTORIES_CREATED); - touch(fs, src); - fs.rename(src, dest); - Object summary = new Object() { - @Override - public String toString() { - return String.format("[%s, %s, %s, %s]", - directoriesCreated, directoriesDeleted, - deleteRequests, fakeDirectoriesDeleted); - } - }; - - String state = String.format("after touch(%s) %s", - src, summary); - touch(fs, src); - fs.rename(src, dest); - directoriesCreated.assertDiffEquals(state, 0); - - - state = String.format("after rename(%s, %s) %s", - src, dest, summary); - // here we expect there to be no fake directories - directoriesCreated.assertDiffEquals(state, 0); - // one for the renamed file only - deleteRequests.assertDiffEquals(state, 1); - directoriesDeleted.assertDiffEquals(state, 0); - fakeDirectoriesDeleted.assertDiffEquals(state, 0); - - // delete that destination file, assert only the file delete was issued - reset(deleteRequests, directoriesCreated, directoriesDeleted, - fakeDirectoriesDeleted); - - fs.delete(dest, false); - // here we expect there to be no fake directories - directoriesCreated.assertDiffEquals(state, 0); - // one for the deleted file - deleteRequests.assertDiffEquals(state, 1); - directoriesDeleted.assertDiffEquals(state, 0); - fakeDirectoriesDeleted.assertDiffEquals(state, 0); - } finally { - fs.delete(src, false); - fs.delete(dest, false); - } - } + // a LIST will find it and declare as empty + S3AFileStatus status = verifyRawInnerGetFileStatus(emptydir, true, + StatusProbeEnum.LIST_ONLY, + FILE_STATUS_DIR_PROBE); + assertEmptyDirStatus(status, Tristate.TRUE); - @Test - public void testDirProbes() throws Throwable { - describe("Test directory probe cost -raw only"); - S3AFileSystem fs = getFileSystem(); - assume("Unguarded FS only", !fs.hasMetadataStore()); - String dir = "testEmptyDirHeadProbe"; - Path emptydir = path(dir); - // Create the empty directory. - fs.mkdirs(emptydir); - - // metrics and assertions. - resetMetricDiffs(); - - intercept(FileNotFoundException.class, () -> - fs.innerGetFileStatus(emptydir, false, - StatusProbeEnum.HEAD_ONLY)); - verifyOperationCount(1, 0); - - // a LIST will find it -but it doesn't consider it an empty dir. - S3AFileStatus status = fs.innerGetFileStatus(emptydir, true, - StatusProbeEnum.LIST_ONLY); - verifyOperationCount(0, 1); - Assertions.assertThat(status) - .describedAs("LIST output is not considered empty") - .matches(s -> !s.isEmptyDirectory().equals(Tristate.TRUE), "is empty"); - - // finally, skip all probes and expect no operations toThere are - // take place - intercept(FileNotFoundException.class, () -> - fs.innerGetFileStatus(emptydir, false, - EnumSet.noneOf(StatusProbeEnum.class))); - verifyOperationCount(0, 0); + // skip all probes and expect no operations to take place + interceptRawGetFileStatusFNFE(emptydir, false, + EnumSet.noneOf(StatusProbeEnum.class), + NO_IO); // now add a trailing slash to the key and use the // deep internal s3GetFileStatus method call. String emptyDirTrailingSlash = fs.pathToKey(emptydir.getParent()) - + "/" + dir + "/"; + + "/" + emptydir.getName() + "/"; // A HEAD request does not probe for keys with a trailing / - intercept(FileNotFoundException.class, () -> + interceptRaw(FileNotFoundException.class, "", + NO_IO, () -> fs.s3GetFileStatus(emptydir, emptyDirTrailingSlash, - StatusProbeEnum.HEAD_ONLY, null)); - verifyOperationCount(0, 0); + StatusProbeEnum.HEAD_ONLY, null, false)); // but ask for a directory marker and you get the entry - status = fs.s3GetFileStatus(emptydir, - emptyDirTrailingSlash, - StatusProbeEnum.DIR_MARKER_ONLY, null); - verifyOperationCount(1, 0); + status = verifyRaw(FILE_STATUS_DIR_PROBE, () -> + fs.s3GetFileStatus(emptydir, + emptyDirTrailingSlash, + StatusProbeEnum.LIST_ONLY, + null, + true)); assertEquals(emptydir, status.getPath()); + assertEmptyDirStatus(status, Tristate.TRUE); } @Test - public void testCreateCost() throws Throwable { - describe("Test file creation cost -raw only"); + public void testNeedEmptyDirectoryProbeRequiresList() throws Throwable { S3AFileSystem fs = getFileSystem(); - assume("Unguarded FS only", !fs.hasMetadataStore()); - resetMetricDiffs(); - Path testFile = path("testCreateCost"); + intercept(IllegalArgumentException.class, "", () -> + fs.s3GetFileStatus(new Path("/something"), "/something", + StatusProbeEnum.HEAD_ONLY, null, true)); + } + @Test + public void testCreateCost() throws Throwable { + describe("Test file creation cost -raw only"); + assumeUnguarded(); + Path testFile = methodPath(); // when overwrite is false, the path is checked for existence. - try (FSDataOutputStream out = fs.create(testFile, false)) { - verifyOperationCount(2, 1); - } - + create(testFile, false, + CREATE_FILE_NO_OVERWRITE); // but when true: only the directory checks take place. - try (FSDataOutputStream out = fs.create(testFile, true)) { - verifyOperationCount(1, 1); - } + create(testFile, true, CREATE_FILE_OVERWRITE); + } + + @Test + public void testCreateCostFileExists() throws Throwable { + describe("Test cost of create file failing with existing file"); + assumeUnguarded(); + Path testFile = file(methodPath()); + + // now there is a file there, an attempt with overwrite == false will + // fail on the first HEAD. + interceptRaw(FileAlreadyExistsException.class, "", + FILE_STATUS_FILE_PROBE, + () -> file(testFile, false)); + } + + @Test + public void testCreateCostDirExists() throws Throwable { + describe("Test cost of create file failing with existing dir"); + assumeUnguarded(); + Path testFile = dir(methodPath()); + + // now there is a file there, an attempt with overwrite == false will + // fail on the first HEAD. + interceptRaw(FileAlreadyExistsException.class, "", + GET_FILE_STATUS_ON_DIR_MARKER, + () -> file(testFile, false)); + } + /** + * Use the builder API. + * This always looks for a parent unless the caller says otherwise. + */ + @Test + public void testCreateBuilder() throws Throwable { + describe("Test builder file creation cost -raw only"); + assumeUnguarded(); + Path testFile = methodPath(); + dir(testFile.getParent()); + + // builder defaults to looking for parent existence (non-recursive) + buildFile(testFile, false, false, + GET_FILE_STATUS_FNFE // destination file + .plus(FILE_STATUS_DIR_PROBE)); // parent dir + // recursive = false and overwrite=true: + // only make sure the dest path isn't a directory. + buildFile(testFile, true, true, + FILE_STATUS_DIR_PROBE); + + // now there is a file there, an attempt with overwrite == false will + // fail on the first HEAD. + interceptRaw(FileAlreadyExistsException.class, "", + GET_FILE_STATUS_ON_FILE, + () -> buildFile(testFile, false, true, + GET_FILE_STATUS_ON_FILE)); } @Test @@ -656,15 +400,15 @@ public void testCostOfGlobStatus() throws Throwable { // create a bunch of files int filesToCreate = 10; for (int i = 0; i < filesToCreate; i++) { - try (FSDataOutputStream out = fs.create(basePath.suffix("/" + i))) { - verifyOperationCount(1, 1); - } + create(basePath.suffix("/" + i)); } fs.globStatus(basePath.suffix("/*")); // 2 head + 1 list from getFileStatus on path, // plus 1 list to match the glob pattern - verifyOperationCount(2, 2); + verifyRaw(GET_FILE_STATUS_ON_DIR + .plus(LIST_OPERATION), + () -> fs.globStatus(basePath.suffix("/*"))); } @Test @@ -678,14 +422,14 @@ public void testCostOfGlobStatusNoSymlinkResolution() throws Throwable { // create a single file, globStatus returning a single file on a pattern // triggers attempts at symlinks resolution if configured String fileName = "/notASymlinkDOntResolveMeLikeOne"; - try (FSDataOutputStream out = fs.create(basePath.suffix(fileName))) { - verifyOperationCount(1, 1); - } - - fs.globStatus(basePath.suffix("/*")); + create(basePath.suffix(fileName)); // unguarded: 2 head + 1 list from getFileStatus on path, // plus 1 list to match the glob pattern // no additional operations from symlink resolution - verifyOperationCount(2, 2); + verifyRaw(GET_FILE_STATUS_ON_DIR + .plus(LIST_OPERATION), + () -> fs.globStatus(basePath.suffix("/*"))); } + + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARemoteFileChanged.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARemoteFileChanged.java index 3fd70be931997..01a14ef8e9300 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARemoteFileChanged.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARemoteFileChanged.java @@ -326,7 +326,7 @@ protected Path path() throws IOException { * @return a number >= 0. */ private int getFileStatusHeadCount() { - return authMode ? 0 : 1; + return authMode ? 0 : 0; } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardEmptyDirs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardEmptyDirs.java index ab81491c4cf90..db3c2b6c27462 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardEmptyDirs.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardEmptyDirs.java @@ -26,6 +26,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3ObjectSummary; import org.assertj.core.api.Assertions; @@ -57,6 +58,10 @@ */ public class ITestS3GuardEmptyDirs extends AbstractS3ATestBase { + /** + * Rename an empty directory, verify that the empty dir + * marker moves in both S3Guard and in the S3A FS. + */ @Test public void testRenameEmptyDir() throws Throwable { S3AFileSystem fs = getFileSystem(); @@ -67,7 +72,7 @@ public void testRenameEmptyDir() throws Throwable { String destDirMarker = fs.pathToKey(destDir) + "/"; // set things up. mkdirs(sourceDir); - // there'a source directory marker + // there's source directory marker fs.getObjectMetadata(sourceDirMarker); S3AFileStatus srcStatus = getEmptyDirStatus(sourceDir); assertEquals("Must be an empty dir: " + srcStatus, Tristate.TRUE, @@ -82,8 +87,12 @@ public void testRenameEmptyDir() throws Throwable { () -> getEmptyDirStatus(sourceDir)); // and verify that there's no dir marker hidden under a tombstone intercept(FileNotFoundException.class, - () -> Invoker.once("HEAD", sourceDirMarker, - () -> fs.getObjectMetadata(sourceDirMarker))); + () -> Invoker.once("HEAD", sourceDirMarker, () -> { + ObjectMetadata md = fs.getObjectMetadata(sourceDirMarker); + return String.format("Object %s of length %d", + sourceDirMarker, md.getInstanceLength()); + })); + // the parent dir mustn't be confused S3AFileStatus baseStatus = getEmptyDirStatus(basePath); assertEquals("Must not be an empty dir: " + baseStatus, Tristate.FALSE, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java index 3c67e252e6e69..0246b5415f18f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.contract.s3a.S3AContract; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.collect.Lists; import org.assertj.core.api.Assertions; import org.junit.Assume; @@ -560,24 +561,23 @@ public void testInconsistentS3ClientDeletes() throws Throwable { + " paths"); ListObjectsV2Result postDeleteDelimited = listObjectsV2(fs, key, "/"); - assertListSizeEqual( + boolean stripTombstones = false; + assertObjectSummariesEqual( "InconsistentAmazonS3Client added back objects incorrectly " + "in a non-recursive listing", - preDeleteDelimited.getObjectSummaries(), - postDeleteDelimited.getObjectSummaries()); + preDeleteDelimited, postDeleteDelimited, + stripTombstones); assertListSizeEqual("InconsistentAmazonS3Client added back prefixes incorrectly " + "in a non-recursive listing", preDeleteDelimited.getCommonPrefixes(), - postDeleteDelimited.getCommonPrefixes() - ); + postDeleteDelimited.getCommonPrefixes()); LOG.info("Executing Deep listing"); ListObjectsV2Result postDeleteUndelimited = listObjectsV2(fs, key, null); - assertListSizeEqual("InconsistentAmazonS3Client added back objects incorrectly " + - "in a recursive listing", - preDeleteUndelimited.getObjectSummaries(), - postDeleteUndelimited.getObjectSummaries() - ); + assertObjectSummariesEqual("InconsistentAmazonS3Client added back objects" + + " incorrectly in a recursive listing", + preDeleteUndelimited, postDeleteUndelimited, + stripTombstones); assertListSizeEqual("InconsistentAmazonS3Client added back prefixes incorrectly " + "in a recursive listing", @@ -586,6 +586,24 @@ public void testInconsistentS3ClientDeletes() throws Throwable { ); } + private void assertObjectSummariesEqual(final String message, + final ListObjectsV2Result expected, + final ListObjectsV2Result actual, + final boolean stripTombstones) { + assertCollectionsEqual( + message, + stringify(expected.getObjectSummaries(), stripTombstones), + stringify(actual.getObjectSummaries(), stripTombstones)); + } + + List stringify(List objects, + boolean stripTombstones) { + return objects.stream() + .filter(s -> !stripTombstones || !(s.getKey().endsWith("/"))) + .map(s -> s.getKey()) + .collect(Collectors.toList()); + } + /** * Require the v2 S3 list API. */ @@ -682,6 +700,22 @@ public void testListingReturnsVersionMetadata() throws Throwable { versionId, locatedFileStatus.getVersionId()); } + /** + * Assert that the two collections match using + * object equality of the elements within. + * @param message text for the assertion + * @param expected expected list + * @param actual actual list + * @param type of list + */ + private void assertCollectionsEqual(String message, + Collection expected, + Collection actual) { + Assertions.assertThat(actual) + .describedAs(message) + .containsExactlyInAnyOrderElementsOf(expected); + } + /** * Assert that the two list sizes match; failure message includes the lists. * @param message text for the assertion diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardOutOfBandOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardOutOfBandOperations.java index 32ead7f3fed71..2d4173d1c2ad4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardOutOfBandOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardOutOfBandOperations.java @@ -56,7 +56,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; import static org.apache.hadoop.fs.contract.ContractTestUtils.writeTextFile; import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH; -import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_METADATASTORE_METADATA_TTL; +import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE; +import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE_NONE; import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE; import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_METADATA_TTL; import static org.apache.hadoop.fs.s3a.Constants.RETRY_INTERVAL; @@ -169,12 +170,16 @@ protected Configuration createConfiguration() { RETRY_LIMIT, RETRY_INTERVAL, S3GUARD_CONSISTENCY_RETRY_INTERVAL, - S3GUARD_CONSISTENCY_RETRY_LIMIT); + S3GUARD_CONSISTENCY_RETRY_LIMIT, + CHANGE_DETECT_MODE, + METADATASTORE_METADATA_TTL); conf.setInt(RETRY_LIMIT, 3); conf.setInt(S3GUARD_CONSISTENCY_RETRY_LIMIT, 3); + conf.set(CHANGE_DETECT_MODE, CHANGE_DETECT_MODE_NONE); final String delay = "10ms"; conf.set(RETRY_INTERVAL, delay); conf.set(S3GUARD_CONSISTENCY_RETRY_INTERVAL, delay); + conf.set(METADATASTORE_METADATA_TTL, delay); return conf; } @@ -232,12 +237,13 @@ private S3AFileSystem createGuardedFS(boolean authoritativeMode) URI uri = testFS.getUri(); removeBaseAndBucketOverrides(uri.getHost(), config, + CHANGE_DETECT_MODE, METADATASTORE_AUTHORITATIVE, METADATASTORE_METADATA_TTL, AUTHORITATIVE_PATH); config.setBoolean(METADATASTORE_AUTHORITATIVE, authoritativeMode); config.setLong(METADATASTORE_METADATA_TTL, - DEFAULT_METADATASTORE_METADATA_TTL); + 5_000); final S3AFileSystem gFs = createFS(uri, config); // set back the same metadata store instance gFs.setMetadataStore(realMs); @@ -857,7 +863,7 @@ private void verifyFileStatusAsExpected(final String firstText, expectedLength, guardedLength); } else { assertEquals( - "File length in authoritative table with " + stats, + "File length in non-authoritative table with " + stats, expectedLength, guardedLength); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 118c9ee773a6b..c5670b09c3db5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -87,10 +87,15 @@ public interface S3ATestConstants { */ String KEY_CSVTEST_FILE = S3A_SCALE_TEST + "csvfile"; + /** + * The landsat bucket: {@value}. + */ + String LANDSAT_BUCKET = "s3a://landsat-pds/"; + /** * Default path for the multi MB test file: {@value}. */ - String DEFAULT_CSVTEST_FILE = "s3a://landsat-pds/scene_list.gz"; + String DEFAULT_CSVTEST_FILE = LANDSAT_BUCKET + "scene_list.gz"; /** * Name of the property to define the timeout for scale tests: {@value}. @@ -218,4 +223,10 @@ public interface S3ATestConstants { */ String S3GUARD_DDB_TEST_TABLE_NAME_KEY = "fs.s3a.s3guard.ddb.test.table"; + + /** + * Test option to enable audits of the method path after + * every test case. + */ + String DIRECTORY_MARKER_AUDIT = "fs.s3a.directory.marker.audit"; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index aa5979dbf751e..f225800b872f3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -618,6 +618,14 @@ public static Configuration prepareTestConfiguration(final Configuration conf) { // add this so that even on tests where the FS is shared, // the FS is always "magic" conf.setBoolean(MAGIC_COMMITTER_ENABLED, true); + + // directory marker policy + String directoryRetention = getTestProperty( + conf, + DIRECTORY_MARKER_POLICY, + DEFAULT_DIRECTORY_MARKER_POLICY); + conf.set(DIRECTORY_MARKER_POLICY, directoryRetention); + return conf; } @@ -882,7 +890,8 @@ public static T terminateService(final T service) { public static S3AFileStatus getStatusWithEmptyDirFlag( final S3AFileSystem fs, final Path dir) throws IOException { - return fs.innerGetFileStatus(dir, true, StatusProbeEnum.ALL); + return fs.innerGetFileStatus(dir, true, + StatusProbeEnum.ALL); } /** @@ -1441,4 +1450,26 @@ public static Set getCurrentThreadNames() { .collect(Collectors.toCollection(TreeSet::new)); return threads; } + + /** + * Call the package-private {@code innerGetFileStatus()} method + * on the passed in FS. + * @param fs filesystem + * @param path path + * @param needEmptyDirectoryFlag look for empty directory + * @param probes file status probes to perform + * @return the status + * @throws IOException + */ + public static S3AFileStatus innerGetFileStatus( + S3AFileSystem fs, + Path path, + boolean needEmptyDirectoryFlag, + Set probes) throws IOException { + + return fs.innerGetFileStatus( + path, + needEmptyDirectoryFlag, + probes); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java index e90518a9cbd0f..34a275b580f25 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java @@ -76,11 +76,15 @@ public void testFakeDirectory() throws Exception { String key = path.toUri().getPath().substring(1); when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) .thenThrow(NOT_FOUND); - ObjectMetadata meta = new ObjectMetadata(); - meta.setContentLength(0L); - when(s3.getObjectMetadata(argThat( - correctGetMetadataRequest(BUCKET, key + "/")) - )).thenReturn(meta); + String keyDir = key + "/"; + ListObjectsV2Result listResult = new ListObjectsV2Result(); + S3ObjectSummary objectSummary = new S3ObjectSummary(); + objectSummary.setKey(keyDir); + objectSummary.setSize(0L); + listResult.getObjectSummaries().add(objectSummary); + when(s3.listObjectsV2(argThat( + matchListV2Request(BUCKET, keyDir)) + )).thenReturn(listResult); FileStatus stat = fs.getFileStatus(path); assertNotNull(stat); assertEquals(fs.makeQualified(path), stat.getPath()); @@ -161,4 +165,14 @@ private ArgumentMatcher correctGetMetadataRequest( && request.getBucketName().equals(bucket) && request.getKey().equals(key); } + + private ArgumentMatcher matchListV2Request( + String bucket, String key) { + return (ListObjectsV2Request request) -> { + return request != null + && request.getBucketName().equals(bucket) + && request.getPrefix().equals(key); + }; + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java index a8e7a57057605..2848fb70b6d61 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java @@ -410,8 +410,7 @@ public void checkBasicFileOperations() throws Throwable { // this is HEAD + "/" on S3; get on S3Guard auth when the path exists, - accessDeniedIf(!s3guard, () -> - readonlyFS.listStatus(emptyDir)); + readonlyFS.listStatus(emptyDir); // a recursive list of the no-read-directory works because // there is no directory marker, it becomes a LIST call. @@ -421,14 +420,9 @@ public void checkBasicFileOperations() throws Throwable { // and so working. readonlyFS.getFileStatus(noReadDir); - // empty dir checks work when guarded because even in non-auth mode - // there are no checks for directories being out of date - // without S3, the HEAD path + "/" is blocked - accessDeniedIf(!s3guard, () -> - readonlyFS.getFileStatus(emptyDir)); - + readonlyFS.getFileStatus(emptyDir); // now look at a file; the outcome depends on the mode. - accessDeniedIf(!guardedInAuthMode, () -> + accessDeniedIf(!s3guard, () -> readonlyFS.getFileStatus(subdirFile)); // irrespective of mode, the attempt to read the data will fail. @@ -443,7 +437,7 @@ public void checkBasicFileOperations() throws Throwable { // This means that permissions on the file do not get checked. // See: HADOOP-16464. Optional optIn = accessDeniedIf( - !guardedInAuthMode, () -> readonlyFS.open(emptyFile)); + !s3guard, () -> readonlyFS.open(emptyFile)); if (optIn.isPresent()) { try (FSDataInputStream is = optIn.get()) { Assertions.assertThat(is.read()) @@ -461,17 +455,17 @@ public void checkGlobOperations() throws Throwable { describe("Glob Status operations"); // baseline: the real filesystem on a subdir globFS(getFileSystem(), subdirFile, null, false, 1); - // a file fails if not in auth mode - globFS(readonlyFS, subdirFile, null, !guardedInAuthMode, 1); + // a file fails if not guarded + globFS(readonlyFS, subdirFile, null, !s3guard, 1); // empty directories don't fail. - FileStatus[] st = globFS(readonlyFS, emptyDir, null, !s3guard, 1); + FileStatus[] st = globFS(readonlyFS, emptyDir, null, false, 1); if (s3guard) { assertStatusPathEquals(emptyDir, st); } st = globFS(readonlyFS, noReadWildcard, - null, !s3guard, 2); + null, false, 2); if (s3guard) { Assertions.assertThat(st) .extracting(FileStatus::getPath) @@ -481,12 +475,12 @@ public void checkGlobOperations() throws Throwable { // there is precisely one .docx file (subdir2File2.docx) globFS(readonlyFS, new Path(noReadDir, "*/*.docx"), - null, !s3guard, 1); + null, false, 1); // there are no .doc files. globFS(readonlyFS, new Path(noReadDir, "*/*.doc"), - null, !s3guard, 0); + null, false, 0); globFS(readonlyFS, noReadDir, EVERYTHING, false, 1); // and a filter without any wildcarded pattern only finds @@ -513,17 +507,14 @@ public void checkSingleThreadedLocatedFileStatus() throws Throwable { true, HIDDEN_FILE_FILTER, true); - accessDeniedIf(!s3guard, - () -> fetcher.getFileStatuses()) - .ifPresent(stats -> { - Assertions.assertThat(stats) - .describedAs("result of located scan").flatExtracting(FileStatus::getPath) - .containsExactlyInAnyOrder( - emptyFile, - subdirFile, - subdir2File1, - subdir2File2); - }); + Assertions.assertThat(fetcher.getFileStatuses()) + .describedAs("result of located scan") + .flatExtracting(FileStatus::getPath) + .containsExactlyInAnyOrder( + emptyFile, + subdirFile, + subdir2File1, + subdir2File2); } /** @@ -542,15 +533,11 @@ public void checkLocatedFileStatusFourThreads() throws Throwable { true, EVERYTHING, true); - accessDeniedIf(!s3guard, - () -> fetcher.getFileStatuses()) - .ifPresent(stats -> { - Assertions.assertThat(stats) - .describedAs("result of located scan") - .isNotNull() - .flatExtracting(FileStatus::getPath) - .containsExactlyInAnyOrder(subdirFile, subdir2File1); - }); + Assertions.assertThat(fetcher.getFileStatuses()) + .describedAs("result of located scan") + .isNotNull() + .flatExtracting(FileStatus::getPath) + .containsExactlyInAnyOrder(subdirFile, subdir2File1); } /** @@ -567,7 +554,7 @@ public void checkLocatedFileStatusScanFile() throws Throwable { true, TEXT_FILE, true); - accessDeniedIf(!guardedInAuthMode, + accessDeniedIf(!s3guard, () -> fetcher.getFileStatuses()) .ifPresent(stats -> { Assertions.assertThat(stats) @@ -631,19 +618,16 @@ public void checkLocatedFileStatusNonexistentPath() throws Throwable { */ public void checkDeleteOperations() throws Throwable { describe("Testing delete operations"); - - if (!authMode) { - // unguarded or non-auth S3Guard to fail on HEAD + / - accessDenied(() -> readonlyFS.delete(emptyDir, true)); + readonlyFS.delete(emptyDir, true); + if (!s3guard) { // to fail on HEAD accessDenied(() -> readonlyFS.delete(emptyFile, true)); } else { - // auth mode checks DDB for status and then issues the DELETE - readonlyFS.delete(emptyDir, true); + // checks DDB for status and then issues the DELETE readonlyFS.delete(emptyFile, true); } - // this will succeed for both as there is no subdir marker. + // this will succeed for both readonlyFS.delete(subDir, true); // after which it is not there fileNotFound(() -> readonlyFS.getFileStatus(subDir)); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestDirectoryMarkerPolicy.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestDirectoryMarkerPolicy.java new file mode 100644 index 0000000000000..194cd645c0714 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestDirectoryMarkerPolicy.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.util.Arrays; +import java.util.Collection; +import java.util.function.Predicate; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP; + +/** + * Unit tests for directory marker policies. + */ +@RunWith(Parameterized.class) +public class TestDirectoryMarkerPolicy extends AbstractHadoopTestBase { + + @Parameterized.Parameters(name = "{0}") + public static Collection data() { + return Arrays.asList(new Object[][]{ + { + DirectoryPolicy.MarkerPolicy.Delete, + FAIL_IF_INVOKED, + false, false + }, + { + DirectoryPolicy.MarkerPolicy.Keep, + FAIL_IF_INVOKED, + true, true + }, + { + DirectoryPolicy.MarkerPolicy.Authoritative, + AUTH_PATH_ONLY, + false, true + } + }); + } + + private final DirectoryPolicy directoryPolicy; + + private final boolean expectNonAuthDelete; + + private final boolean expectAuthDelete; + + public TestDirectoryMarkerPolicy( + final DirectoryPolicy.MarkerPolicy markerPolicy, + final Predicate authoritativeness, + final boolean expectNonAuthDelete, + final boolean expectAuthDelete) { + this.directoryPolicy = newPolicy(markerPolicy, authoritativeness); + this.expectNonAuthDelete = expectNonAuthDelete; + this.expectAuthDelete = expectAuthDelete; + } + + /** + * Create a new retention policy. + * @param markerPolicy policy option + * @param authoritativeness predicate for determining if + * a path is authoritative. + * @return the retention policy. + */ + private DirectoryPolicy newPolicy( + DirectoryPolicy.MarkerPolicy markerPolicy, + Predicate authoritativeness) { + return new DirectoryPolicyImpl(markerPolicy, authoritativeness); + } + + private static final Predicate AUTH_PATH_ONLY = + (p) -> p.toUri().getPath().startsWith("/auth/"); + + private static final Predicate FAIL_IF_INVOKED = (p) -> { + throw new RuntimeException("failed"); + }; + + private final Path nonAuthPath = new Path("s3a://bucket/nonauth/data"); + + private final Path authPath = new Path("s3a://bucket/auth/data1"); + + private final Path deepAuth = new Path("s3a://bucket/auth/d1/d2/data2"); + + /** + * Assert that a path has a retention outcome. + * @param path path + * @param retain should the marker be retained + */ + private void assertMarkerRetention(Path path, boolean retain) { + Assertions.assertThat(directoryPolicy.keepDirectoryMarkers(path)) + .describedAs("Retention of path %s by %s", path, directoryPolicy) + .isEqualTo(retain); + } + + /** + * Assert that a path has a capability. + */ + private void assertPathCapability(Path path, + String capability, + boolean outcome) { + Assertions.assertThat(directoryPolicy) + .describedAs("%s support for capability %s by path %s" + + " expected as %s", + directoryPolicy, capability, path, outcome) + .matches(p -> p.hasPathCapability(path, capability) == outcome, + "pathCapability"); + } + + @Test + public void testNonAuthPath() throws Throwable { + assertMarkerRetention(nonAuthPath, expectNonAuthDelete); + assertPathCapability(nonAuthPath, + STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE, + !expectNonAuthDelete); + assertPathCapability(nonAuthPath, + STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP, + expectNonAuthDelete); + } + + @Test + public void testAuthPath() throws Throwable { + assertMarkerRetention(authPath, expectAuthDelete); + assertPathCapability(authPath, + STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE, + !expectAuthDelete); + assertPathCapability(authPath, + STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP, + expectAuthDelete); + } + + @Test + public void testDeepAuthPath() throws Throwable { + assertMarkerRetention(deepAuth, expectAuthDelete); + assertPathCapability(deepAuth, + STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE, + !expectAuthDelete); + assertPathCapability(deepAuth, + STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP, + expectAuthDelete); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java new file mode 100644 index 0000000000000..db0542ddc94a6 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java @@ -0,0 +1,637 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.Callable; + +import org.assertj.core.api.Assertions; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.Statistic; +import org.apache.hadoop.fs.s3a.Tristate; +import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; + +import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; +import static org.apache.hadoop.fs.s3a.Statistic.*; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; +import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.expect; +import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe; +import static org.apache.hadoop.test.AssertExtensions.dynamicDescription; + +/** + * Abstract class for tests which make assertions about cost. + *

+ * Factored out from {@code ITestS3AFileOperationCost} + */ +public class AbstractS3ACostTest extends AbstractS3ATestBase { + + /** + * Parameter: should the stores be guarded? + */ + private final boolean s3guard; + + /** + * Parameter: should directory markers be retained? + */ + private final boolean keepMarkers; + + /** + * Is this an auth mode test run? + */ + private final boolean authoritative; + + /** probe states calculated from the configuration options. */ + private boolean isGuarded; + + private boolean isRaw; + + private boolean isAuthoritative; + + private boolean isNonAuth; + + private boolean isKeeping; + + private boolean isDeleting; + + private OperationCostValidator costValidator; + + public AbstractS3ACostTest( + final boolean s3guard, + final boolean keepMarkers, + final boolean authoritative) { + this.s3guard = s3guard; + this.keepMarkers = keepMarkers; + this.authoritative = authoritative; + } + + @Override + public Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + String bucketName = getTestBucketName(conf); + removeBucketOverrides(bucketName, conf, + S3_METADATA_STORE_IMPL); + if (!isGuarded()) { + // in a raw run remove all s3guard settings + removeBaseAndBucketOverrides(bucketName, conf, + S3_METADATA_STORE_IMPL); + } + removeBaseAndBucketOverrides(bucketName, conf, + DIRECTORY_MARKER_POLICY, + METADATASTORE_AUTHORITATIVE, + AUTHORITATIVE_PATH); + // directory marker options + conf.set(DIRECTORY_MARKER_POLICY, + keepMarkers + ? DIRECTORY_MARKER_POLICY_KEEP + : DIRECTORY_MARKER_POLICY_DELETE); + conf.setBoolean(METADATASTORE_AUTHORITATIVE, authoritative); + disableFilesystemCaching(conf); + return conf; + } + + @Override + public void setup() throws Exception { + super.setup(); + if (isGuarded()) { + // s3guard is required for those test runs where any of the + // guard options are set + assumeS3GuardState(true, getConfiguration()); + } + S3AFileSystem fs = getFileSystem(); + skipDuringFaultInjection(fs); + + // build up the states + isGuarded = isGuarded(); + + isRaw = !isGuarded; + isAuthoritative = isGuarded && authoritative; + isNonAuth = isGuarded && !authoritative; + + isKeeping = isKeepingMarkers(); + + isDeleting = !isKeeping; + + // insert new metrics so as to keep the list sorted + costValidator = OperationCostValidator.builder(getFileSystem()) + .withMetrics( + DIRECTORIES_CREATED, + DIRECTORIES_DELETED, + FAKE_DIRECTORIES_DELETED, + FILES_DELETED, + INVOCATION_COPY_FROM_LOCAL_FILE, + OBJECT_COPY_REQUESTS, + OBJECT_DELETE_REQUESTS, + OBJECT_LIST_REQUESTS, + OBJECT_METADATA_REQUESTS, + OBJECT_PUT_BYTES, + OBJECT_PUT_REQUESTS) + .build(); + } + + public void assumeUnguarded() { + assume("Unguarded FS only", !isGuarded()); + } + + /** + * Is the store guarded authoritatively on the test path? + * @return true if the condition is met on this test run. + */ + public boolean isAuthoritative() { + return authoritative; + } + + /** + * Is the store guarded? + * @return true if the condition is met on this test run. + */ + public boolean isGuarded() { + return s3guard; + } + + /** + * Is the store raw? + * @return true if the condition is met on this test run. + */ + public boolean isRaw() { + return isRaw; + } + + /** + * Is the store guarded non-authoritatively on the test path? + * @return true if the condition is met on this test run. + */ + public boolean isNonAuth() { + return isNonAuth; + } + + public boolean isDeleting() { + return isDeleting; + } + + public boolean isKeepingMarkers() { + return keepMarkers; + } + + /** + * A special object whose toString() value is the current + * state of the metrics. + */ + protected Object getMetricSummary() { + return costValidator; + } + + /** + * Create then close the file through the builder API. + * @param path path + * @param overwrite overwrite flag + * @param recursive true == skip parent existence check + * @param cost expected cost + * @return path to new object. + */ + protected Path buildFile(Path path, + boolean overwrite, + boolean recursive, + OperationCost cost) throws Exception { + resetStatistics(); + verifyRaw(cost, () -> { + FSDataOutputStreamBuilder builder = getFileSystem().createFile(path) + .overwrite(overwrite); + if (recursive) { + builder.recursive(); + } + FSDataOutputStream stream = builder.build(); + stream.close(); + return stream.toString(); + }); + return path; + } + + /** + * Create a directory, returning its path. + * @param p path to dir. + * @return path of new dir + */ + protected Path dir(Path p) throws IOException { + mkdirs(p); + return p; + } + + /** + * Create a file, returning its path. + * @param p path to file. + * @return path of new file + */ + protected Path file(Path p) throws IOException { + return file(p, true); + } + + /** + * Create a file, returning its path. + * @param path path to file. + * @param overwrite overwrite flag + * @return path of new file + */ + protected Path file(Path path, final boolean overwrite) + throws IOException { + getFileSystem().create(path, overwrite).close(); + return path; + } + + /** + * Touch a file, overwriting. + * @param path path + * @return path to new object. + */ + protected Path create(Path path) throws Exception { + return create(path, true, CREATE_FILE_OVERWRITE); + } + + /** + * Create then close the file. + * @param path path + * @param overwrite overwrite flag + * @param cost expected cost + + * @return path to new object. + */ + protected Path create(Path path, boolean overwrite, + OperationCost cost) throws Exception { + return verifyRaw(cost, () -> + file(path, overwrite)); + } + + /** + * Execute rename, returning the current metrics. + * For use in l-expressions. + * @param source source path. + * @param dest dest path + * @return a string for exceptions. + */ + public String execRename(final Path source, + final Path dest) throws IOException { + getFileSystem().rename(source, dest); + return String.format("rename(%s, %s): %s", + dest, source, getMetricSummary()); + } + + /** + * How many directories are in a path? + * @param path path to probe. + * @return the number of entries below root this path is + */ + protected int directoriesInPath(Path path) { + return path.isRoot() ? 0 : 1 + directoriesInPath(path.getParent()); + } + + /** + * Reset all the metrics being tracked. + */ + private void resetStatistics() { + costValidator.resetMetricDiffs(); + } + + /** + * Execute a closure and verify the metrics. + * @param eval closure to evaluate + * @param expected varargs list of expected diffs + * @param return type. + * @return the result of the evaluation + */ + protected T verifyMetrics( + Callable eval, + OperationCostValidator.ExpectedProbe... expected) throws Exception { + return costValidator.exec(eval, expected); + + } + + /** + * Execute a closure, expecting an exception. + * Verify the metrics after the exception has been caught and + * validated. + * @param clazz type of exception + * @param text text to look for in exception (optional) + * @param eval closure to evaluate + * @param expected varargs list of expected diffs + * @param return type of closure + * @param exception type + * @return the exception caught. + * @throws Exception any other exception + */ + protected E verifyMetricsIntercepting( + Class clazz, + String text, + Callable eval, + OperationCostValidator.ExpectedProbe... expected) throws Exception { + return costValidator.intercepting(clazz, text, eval, expected); + } + + /** + * Execute a closure expecting an exception. + * @param clazz type of exception + * @param text text to look for in exception (optional) + * @param head expected head request count. + * @param list expected list request count. + * @param eval closure to evaluate + * @param return type of closure + * @param exception type + * @return the exception caught. + * @throws Exception any other exception + */ + protected E interceptRaw( + Class clazz, + String text, + OperationCost cost, + Callable eval) throws Exception { + return verifyMetricsIntercepting(clazz, text, eval, whenRaw(cost)); + } + + /** + * Declare the expected cost on any FS. + * @param cost costs to expect + * @return a probe. + */ + protected OperationCostValidator.ExpectedProbe always( + OperationCost cost) { + return expect(true, cost); + } + + /** + * Declare the expected cost on a raw FS. + * @param cost costs to expect + * @return a probe. + */ + protected OperationCostValidator.ExpectedProbe whenRaw( + OperationCost cost) { + return expect(isRaw(), cost); + } + + /** + * Declare the expected cost on a guarded FS. + * @param cost costs to expect + * @return a probe. + */ + protected OperationCostValidator.ExpectedProbe whenGuarded( + OperationCost cost) { + return expect(isGuarded(), cost); + } + + /** + * Declare the expected cost on a guarded auth FS. + * @param cost costs to expect + * @return a probe. + */ + protected OperationCostValidator.ExpectedProbe whenAuthoritative( + OperationCost cost) { + return expect(isAuthoritative(), cost); + } + + + /** + * Declare the expected cost on a guarded nonauth FS. + * @param cost costs to expect + * @return a probe. + */ + protected OperationCostValidator.ExpectedProbe whenNonauth( + OperationCost cost) { + return expect(isNonAuth(), cost); + } + + + /** + * A metric diff which must hold when the fs is keeping markers. + * @param cost expected cost + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe whenKeeping( + OperationCost cost) { + return expect(isKeepingMarkers(), cost); + } + + /** + * A metric diff which must hold when the fs is keeping markers. + * @param cost expected cost + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe whenDeleting( + OperationCost cost) { + return expect(isDeleting(), cost); + } + + /** + * Execute a closure expecting a specific number of HEAD/LIST calls + * on raw S3 stores only. + * @param cost expected cost + * @param eval closure to evaluate + * @param return type of closure + * @return the result of the evaluation + */ + protected T verifyRaw( + OperationCost cost, + Callable eval) throws Exception { + return verifyMetrics(eval, whenRaw(cost)); + } + + /** + * Execute {@code S3AFileSystem#innerGetFileStatus(Path, boolean, Set)} + * for the given probes. + * expect the specific HEAD/LIST count with a raw FS. + * @param path path + * @param needEmptyDirectoryFlag look for empty directory + * @param probes file status probes to perform + * @param cost expected cost + * @return the status + */ + public S3AFileStatus verifyRawInnerGetFileStatus( + Path path, + boolean needEmptyDirectoryFlag, + Set probes, + OperationCost cost) throws Exception { + return verifyRaw(cost, () -> + innerGetFileStatus(getFileSystem(), + path, + needEmptyDirectoryFlag, + probes)); + } + + /** + * Execute {@code S3AFileSystem#innerGetFileStatus(Path, boolean, Set)} + * for the given probes -expect a FileNotFoundException, + * and the specific HEAD/LIST count with a raw FS. + * @param path path + * @param needEmptyDirectoryFlag look for empty directory + * @param probes file status probes to perform + * @param cost expected cost + */ + + public void interceptRawGetFileStatusFNFE( + Path path, + boolean needEmptyDirectoryFlag, + Set probes, + OperationCost cost) throws Exception { + interceptRaw(FileNotFoundException.class, "", + cost, () -> + innerGetFileStatus(getFileSystem(), + path, + needEmptyDirectoryFlag, + probes)); + } + + /** + * Probe for a path being a directory. + * Metrics are only checked on unguarded stores. + * @param path path + * @param expected expected outcome + * @param cost expected cost on a Raw FS. + */ + protected void isDir(Path path, + boolean expected, + OperationCost cost) throws Exception { + boolean b = verifyRaw(cost, () -> + getFileSystem().isDirectory(path)); + Assertions.assertThat(b) + .describedAs("isDirectory(%s)", path) + .isEqualTo(expected); + } + + /** + * Probe for a path being a file. + * Metrics are only checked on unguarded stores. + * @param path path + * @param expected expected outcome + * @param cost expected cost on a Raw FS. + */ + protected void isFile(Path path, + boolean expected, + OperationCost cost) throws Exception { + boolean b = verifyRaw(cost, () -> + getFileSystem().isFile(path)); + Assertions.assertThat(b) + .describedAs("isFile(%s)", path) + .isEqualTo(expected); + } + + /** + * A metric diff which must always hold. + * @param stat metric source + * @param expected expected value. + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe with( + final Statistic stat, final int expected) { + return probe(stat, expected); + } + + /** + * A metric diff which must hold when the fs is unguarded. + * @param stat metric source + * @param expected expected value. + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe withWhenRaw( + final Statistic stat, final int expected) { + return probe(isRaw(), stat, expected); + } + + /** + * A metric diff which must hold when the fs is guarded. + * @param stat metric source + * @param expected expected value. + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe withWhenGuarded( + final Statistic stat, + final int expected) { + return probe(isGuarded(), stat, expected); + } + + /** + * A metric diff which must hold when the fs is guarded + authoritative. + * @param stat metric source + * @param expected expected value. + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe withWhenAuthoritative( + final Statistic stat, + final int expected) { + return probe(isAuthoritative(), stat, expected); + } + + /** + * A metric diff which must hold when the fs is guarded + authoritative. + * @param stat metric source + * @param expected expected value. + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe withWhenNonauth( + final Statistic stat, + final int expected) { + return probe(isNonAuth(), stat, expected); + } + + /** + * A metric diff which must hold when the fs is keeping markers. + * @param stat metric source + * @param expected expected value. + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe withWhenKeeping( + final Statistic stat, + final int expected) { + return probe(isKeepingMarkers(), stat, expected); + } + + /** + * A metric diff which must hold when the fs is keeping markers. + * @param stat metric source + * @param expected expected value. + * @return the diff. + */ + protected OperationCostValidator.ExpectedProbe withWhenDeleting( + final Statistic stat, + final int expected) { + return probe(isDeleting(), stat, expected); + } + + /** + * Assert the empty directory status of a file is as expected. + * The raised assertion message includes a list of the path. + * @param status status to probe. + * @param expected expected value + */ + protected void assertEmptyDirStatus(final S3AFileStatus status, + final Tristate expected) { + Assertions.assertThat(status.isEmptyDirectory()) + .describedAs(dynamicDescription(() -> + "FileStatus says directory is not empty: " + status + + "\n" + ContractTestUtils.ls( + getFileSystem(), status.getPath()))) + .isEqualTo(expected); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java new file mode 100644 index 0000000000000..ed56802ddfec1 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java @@ -0,0 +1,824 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ObjectMetadata; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3AUtils; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; +import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP; +import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE; +import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * This is a test suite designed to verify that directory markers do + * not get misconstrued as empty directories during operations + * which explicitly or implicitly list directory trees. + *

+ * It is also intended it to be backported to all releases + * which are enhanced to read directory trees where markers have + * been retained. + * Hence: it does not use any of the new helper classes to + * measure the cost of operations or attempt to create markers + * through the FS APIs. + *

+ * Instead, the directory structure to test is created through + * low-level S3 SDK API calls. + * We also skip any probes to measure/assert metrics. + * We're testing the semantics here, not the cost of the operations. + * Doing that makes it a lot easier to backport. + * + *

+ * Similarly: JUnit assertions over AssertJ. + *

+ * The tests work with unguarded buckets only -the bucket settings are changed + * appropriately. + */ +@RunWith(Parameterized.class) +public class ITestDirectoryMarkerListing extends AbstractS3ATestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestDirectoryMarkerListing.class); + + private static final String FILENAME = "fileUnderMarker"; + + private static final String HELLO = "hello"; + + private static final String MARKER = "marker"; + + private static final String MARKER_PEER = "markerpeer"; + + /** + * Parameterization. + */ + @Parameterized.Parameters(name = "{0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {"keep-markers", true}, + {"delete-markers", false}, + }); + } + + /** + * Does rename copy markers? + * Value: {@value} + *

+ * Older releases: yes. + *

+ * The full marker-optimized releases: no. + */ + private static final boolean RENAME_COPIES_MARKERS = false; + + /** + * Test configuration name. + */ + private final String name; + + /** + * Does this test configuration keep markers? + */ + private final boolean keepMarkers; + + /** + * Is this FS deleting markers? + */ + private final boolean isDeletingMarkers; + + /** + * Path to a directory which has a marker. + */ + private Path markerDir; + + /** + * Key to the object representing {@link #markerDir}. + */ + private String markerKey; + + /** + * Key to the object representing {@link #markerDir} with + * a trailing / added. This references the actual object + * which has been created. + */ + private String markerKeySlash; + + /** + * bucket of tests. + */ + private String bucket; + + /** + * S3 Client of the FS. + */ + private AmazonS3 s3client; + + /** + * Path to a file under the marker. + */ + private Path filePathUnderMarker; + + /** + * Key to a file under the marker. + */ + private String fileKeyUnderMarker; + + /** + * base path for the test files; the marker dir goes under this. + */ + private Path basePath; + + /** + * Path to a file a peer of markerDir. + */ + private Path markerPeer; + + /** + * Key to a file a peer of markerDir. + */ + private String markerPeerKey; + + public ITestDirectoryMarkerListing(final String name, + final boolean keepMarkers) { + this.name = name; + this.keepMarkers = keepMarkers; + this.isDeletingMarkers = !keepMarkers; + } + + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + String bucketName = getTestBucketName(conf); + + // Turn off S3Guard + removeBaseAndBucketOverrides(bucketName, conf, + S3_METADATA_STORE_IMPL, + METADATASTORE_AUTHORITATIVE, + AUTHORITATIVE_PATH); + + // directory marker options + removeBaseAndBucketOverrides(bucketName, conf, + DIRECTORY_MARKER_POLICY); + conf.set(DIRECTORY_MARKER_POLICY, + keepMarkers + ? DIRECTORY_MARKER_POLICY_KEEP + : DIRECTORY_MARKER_POLICY_DELETE); + return conf; + } + + /** + * The setup phase includes creating the test objects. + */ + @Override + public void setup() throws Exception { + super.setup(); + S3AFileSystem fs = getFileSystem(); + assume("unguarded FS only", + !fs.hasMetadataStore()); + s3client = fs.getAmazonS3ClientForTesting("markers"); + + bucket = fs.getBucket(); + Path base = new Path(methodPath(), "base"); + + createTestObjects(base); + } + + /** + * Teardown deletes the objects created before + * the superclass does the directory cleanup. + */ + @Override + public void teardown() throws Exception { + if (s3client != null) { + deleteObject(markerKey); + deleteObject(markerKeySlash); + deleteObject(markerPeerKey); + deleteObject(fileKeyUnderMarker); + } + // do this ourselves to avoid audits teardown failing + // when surplus markers are found + deleteTestDirInTeardown(); + super.teardown(); + } + + /** + * Create the test objects under the given path, setting + * various fields in the process. + * @param path parent path of everything + */ + private void createTestObjects(final Path path) throws Exception { + S3AFileSystem fs = getFileSystem(); + basePath = path; + markerDir = new Path(basePath, MARKER); + // peer path has the same initial name to make sure there + // is no confusion there. + markerPeer = new Path(basePath, MARKER_PEER); + markerPeerKey = fs.pathToKey(markerPeer); + markerKey = fs.pathToKey(markerDir); + markerKeySlash = markerKey + "/"; + fileKeyUnderMarker = markerKeySlash + FILENAME; + filePathUnderMarker = new Path(markerDir, FILENAME); + // put the empty dir + fs.mkdirs(markerDir); + touch(fs, markerPeer); + put(fileKeyUnderMarker, HELLO); + } + + /* + ================================================================= + Basic probes + ================================================================= + */ + + @Test + public void testMarkerExists() throws Throwable { + describe("Verify the marker exists"); + head(markerKeySlash); + assertIsDirectory(markerDir); + } + + @Test + public void testObjectUnderMarker() throws Throwable { + describe("verify the file under the marker dir exists"); + assertIsFile(filePathUnderMarker); + head(fileKeyUnderMarker); + } + + /* + ================================================================= + The listing operations + ================================================================= + */ + + @Test + public void testListStatusMarkerDir() throws Throwable { + describe("list the marker directory and expect to see the file"); + assertContainsFileUnderMarkerOnly( + toList(getFileSystem().listStatus(markerDir))); + } + + + @Test + public void testListFilesMarkerDirFlat() throws Throwable { + assertContainsFileUnderMarkerOnly(toList( + getFileSystem().listFiles(markerDir, false))); + } + + @Test + public void testListFilesMarkerDirRecursive() throws Throwable { + List statuses = toList( + getFileSystem().listFiles(markerDir, true)); + assertContainsFileUnderMarkerOnly(statuses); + } + + /** + * Path listing above the base dir MUST only find the file + * and not the marker. + */ + @Test + public void testListStatusBaseDirRecursive() throws Throwable { + List statuses = toList( + getFileSystem().listFiles(basePath, true)); + assertContainsExactlyStatusOfPaths(statuses, filePathUnderMarker, + markerPeer); + } + + @Test + public void testGlobStatusBaseDirRecursive() throws Throwable { + Path escapedPath = new Path(escape(basePath.toUri().getPath())); + List statuses = + exec("glob", () -> + toList(getFileSystem().globStatus(new Path(escapedPath, "*")))); + assertContainsExactlyStatusOfPaths(statuses, markerDir, markerPeer); + assertIsFileAtPath(markerPeer, statuses.get(1)); + } + + @Test + public void testGlobStatusMarkerDir() throws Throwable { + Path escapedPath = new Path(escape(markerDir.toUri().getPath())); + List statuses = + exec("glob", () -> + toList(getFileSystem().globStatus(new Path(escapedPath, "*")))); + assertContainsFileUnderMarkerOnly(statuses); + } + + /** + * Call {@code listLocatedStatus(basePath)} + *

+ * The list here returns the marker peer before the + * dir. Reason: the listing iterators return + * the objects before the common prefixes, and the + * marker dir is coming back as a prefix. + */ + @Test + public void testListLocatedStatusBaseDir() throws Throwable { + List statuses = + exec("listLocatedStatus", () -> + toList(getFileSystem().listLocatedStatus(basePath))); + + assertContainsExactlyStatusOfPaths(statuses, markerPeer, markerDir); + } + + /** + * Call {@code listLocatedStatus(markerDir)}; expect + * the file entry only. + */ + @Test + public void testListLocatedStatusMarkerDir() throws Throwable { + List statuses = + exec("listLocatedStatus", () -> + toList(getFileSystem().listLocatedStatus(markerDir))); + + assertContainsFileUnderMarkerOnly(statuses); + } + + + /* + ================================================================= + Creation Rejection + ================================================================= + */ + + @Test + public void testCreateNoOverwriteMarkerDir() throws Throwable { + describe("create no-overwrite over the marker dir fails"); + head(markerKeySlash); + intercept(FileAlreadyExistsException.class, () -> + exec("create", () -> + getFileSystem().create(markerDir, false))); + // dir is still there. + head(markerKeySlash); + } + + @Test + public void testCreateNoOverwriteFile() throws Throwable { + describe("create-no-overwrite on the file fails"); + + head(fileKeyUnderMarker); + intercept(FileAlreadyExistsException.class, () -> + exec("create", () -> + getFileSystem().create(filePathUnderMarker, false))); + assertTestObjectsExist(); + } + + @Test + public void testCreateFileNoOverwrite() throws Throwable { + describe("verify the createFile() API also fails"); + head(fileKeyUnderMarker); + intercept(FileAlreadyExistsException.class, () -> + exec("create", () -> + getFileSystem().createFile(filePathUnderMarker) + .overwrite(false) + .build())); + assertTestObjectsExist(); + } + + /* + ================================================================= + Delete. + ================================================================= + */ + + @Test + public void testDelete() throws Throwable { + S3AFileSystem fs = getFileSystem(); + // a non recursive delete MUST fail because + // it is not empty + intercept(PathIsNotEmptyDirectoryException.class, () -> + fs.delete(markerDir, false)); + // file is still there + head(fileKeyUnderMarker); + + // recursive delete MUST succeed + fs.delete(markerDir, true); + // and the markers are gone + head404(fileKeyUnderMarker); + head404(markerKeySlash); + // just for completeness + fs.delete(basePath, true); + } + + /* + ================================================================= + Rename. + ================================================================= + */ + + /** + * Rename the base directory, expect the source files to move. + *

+ * Whether or not the marker itself is copied depends on whether + * the release's rename operation explicitly skips + * markers on renames. + */ + @Test + public void testRenameBase() throws Throwable { + describe("rename base directory"); + + Path src = basePath; + Path dest = new Path(methodPath(), "dest"); + assertRenamed(src, dest); + + assertPathDoesNotExist("source", src); + assertPathDoesNotExist("source", filePathUnderMarker); + assertPathExists("dest not found", dest); + + // all the paths dest relative + Path destMarkerDir = new Path(dest, MARKER); + // peer path has the same initial name to make sure there + // is no confusion there. + Path destMarkerPeer = new Path(dest, MARKER_PEER); + String destMarkerKey = toKey(destMarkerDir); + String destMarkerKeySlash = destMarkerKey + "/"; + String destFileKeyUnderMarker = destMarkerKeySlash + FILENAME; + Path destFilePathUnderMarker = new Path(destMarkerDir, FILENAME); + assertIsFile(destFilePathUnderMarker); + assertIsFile(destMarkerPeer); + head(destFileKeyUnderMarker); + + // probe for the marker based on expected rename + // behavior + if (RENAME_COPIES_MARKERS) { + head(destMarkerKeySlash); + } else { + head404(destMarkerKeySlash); + } + + } + + /** + * Rename a file under a marker by passing in the marker + * directory as the destination; the final path is derived + * from the original filename. + *

+ * After the rename: + *
    + *
  1. The data must be at the derived destination path.
    2. + *
  2. The source file must not exist.
    3. + *
  3. The parent dir of the source file must exist.
    4. + *
  4. The marker above the destination file must not exist.
    5. + *
+ */ + @Test + public void testRenameUnderMarkerDir() throws Throwable { + describe("directory rename under an existing marker"); + String file = "sourceFile"; + Path srcDir = new Path(basePath, "srcdir"); + mkdirs(srcDir); + Path src = new Path(srcDir, file); + String srcKey = toKey(src); + put(srcKey, file); + head(srcKey); + + // set the destination to be the marker directory. + Path dest = markerDir; + // rename the source file under the dest dir. + assertRenamed(src, dest); + assertIsFile(new Path(dest, file)); + assertIsDirectory(srcDir); + if (isDeletingMarkers) { + head404(markerKeySlash); + } else { + head(markerKeySlash); + } + } + + /** + * Rename file under a marker, giving the full path to the destination + * file. + *

+ * After the rename: + *
    + *
  1. The data must be at the explicit destination path.
    2. + *
  2. The source file must not exist.
    3. + *
  3. The parent dir of the source file must exist.
    4. + *
  4. The marker above the destination file must not exist.
    5. + *
+ */ + @Test + public void testRenameUnderMarkerWithPath() throws Throwable { + describe("directory rename under an existing marker"); + S3AFileSystem fs = getFileSystem(); + String file = "sourceFile"; + Path srcDir = new Path(basePath, "srcdir"); + mkdirs(srcDir); + Path src = new Path(srcDir, file); + String srcKey = toKey(src); + put(srcKey, file); + head(srcKey); + + // set the destination to be the final file + Path dest = new Path(markerDir, "destFile"); + // rename the source file to the destination file + assertRenamed(src, dest); + assertIsFile(dest); + assertIsDirectory(srcDir); + if (isDeletingMarkers) { + head404(markerKeySlash); + } else { + head(markerKeySlash); + } + } + + /** + * This test creates an empty dir and renames it over the directory marker. + * If the dest was considered to be empty, the rename would fail. + */ + @Test + public void testRenameEmptyDirOverMarker() throws Throwable { + describe("rename an empty directory over the marker"); + S3AFileSystem fs = getFileSystem(); + String dir = "sourceDir"; + Path src = new Path(basePath, dir); + fs.mkdirs(src); + assertIsDirectory(src); + String srcKey = toKey(src) + "/"; + head(srcKey); + Path dest = markerDir; + // renamed into the dest dir + assertFalse("rename(" + src + ", " + dest + ") should have failed", + getFileSystem().rename(src, dest)); + // source is still there + assertIsDirectory(src); + head(srcKey); + // and a non-recursive delete lets us verify it is considered + // an empty dir + assertDeleted(src, false); + assertTestObjectsExist(); + } + + /* + ================================================================= + Utility methods and assertions. + ================================================================= + */ + + /** + * Assert the test objects exist. + */ + private void assertTestObjectsExist() throws Exception { + head(fileKeyUnderMarker); + head(markerKeySlash); + } + + /** + * Put a string to a path. + * @param key key + * @param content string + */ + private void put(final String key, final String content) throws Exception { + exec("PUT " + key, () -> + s3client.putObject(bucket, key, content)); + } + /** + * Delete an object. + * @param key key + * @param content string + */ + private void deleteObject(final String key) throws Exception { + exec("DELETE " + key, () -> { + s3client.deleteObject(bucket, key); + return "deleted " + key; + }); + } + + /** + * Issue a HEAD request. + * @param key + * @return a description of the object. + */ + private String head(final String key) throws Exception { + ObjectMetadata md = exec("HEAD " + key, () -> + s3client.getObjectMetadata(bucket, key)); + return String.format("Object %s of length %d", + key, md.getInstanceLength()); + } + + /** + * Issue a HEAD request and expect a 404 back. + * @param key + * @return the metadata + */ + private void head404(final String key) throws Exception { + intercept(FileNotFoundException.class, "", + "Expected 404 of " + key, () -> + head(key)); + } + + /** + * Execute an operation; transate AWS exceptions. + * @param op operation + * @param call call to make + * @param returned type + * @return result of the call. + * @throws Exception failure + */ + private T exec(String op, Callable call) throws Exception { + ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); + try { + return call.call(); + } catch (AmazonClientException ex) { + throw S3AUtils.translateException(op, "", ex); + } finally { + timer.end(op); + } + } + + /** + * Assert that the listing contains only the status + * of the file under the marker. + * @param statuses status objects + */ + private void assertContainsFileUnderMarkerOnly( + final List statuses) { + + assertContainsExactlyStatusOfPaths(statuses, filePathUnderMarker); + assertIsFileUnderMarker(statuses.get(0)); + } + + /** + * Expect the list of status objects to match that of the paths. + * @param statuses status object list + * @param paths ordered varargs list of paths + * @param type of status objects + */ + private void assertContainsExactlyStatusOfPaths( + List statuses, Path... paths) { + + String actual = statuses.stream() + .map(Object::toString) + .collect(Collectors.joining(";")); + String expected = Arrays.stream(paths) + .map(Object::toString) + .collect(Collectors.joining(";")); + String summary = "expected [" + expected + "]" + + " actual = [" + actual + "]"; + assertEquals("mismatch in size of listing " + summary, + paths.length, statuses.size()); + for (int i = 0; i < statuses.size(); i++) { + assertEquals("Path mismatch at element " + i + " in " + summary, + paths[i], statuses.get(i).getPath()); + } + } + + /** + * Assert the status object refers to the file created + * under the marker. + * @param stat status object + */ + private void assertIsFileUnderMarker(final FileStatus stat) { + assertIsFileAtPath(filePathUnderMarker, stat); + } + + /** + * Assert the status object refers to a path at the given name. + * @param path path + * @param stat status object + */ + private void assertIsFileAtPath(final Path path, final FileStatus stat) { + assertTrue("Is not file " + stat, stat.isFile()); + assertPathEquals(path, stat); + } + + /** + * Assert a status object's path matches expected. + * @param path path to expect + * @param stat status object + */ + private void assertPathEquals(final Path path, final FileStatus stat) { + assertEquals("filename is not the expected path :" + stat, + path, stat.getPath()); + } + + /** + * Given a remote iterator of status objects, + * build a list of the values. + * @param status status list + * @param actual type. + * @return source. + * @throws IOException + */ + private List toList( + RemoteIterator status) throws IOException { + + List l = new ArrayList<>(); + while (status.hasNext()) { + l.add(status.next()); + } + return dump(l); + } + + /** + * Given an array of status objects, + * build a list of the values. + * @param status status list + * @param actual type. + * @return source. + * @throws IOException + */ + private List toList( + T[] status) throws IOException { + return dump(Arrays.asList(status)); + } + + /** + * Dump the string values of a list to the log; return + * the list. + * @param l source. + * @param source type + * @return the list + */ + private List dump(List l) { + int c = 1; + for (T t : l) { + LOG.info("{}\t{}", c++, t); + } + return l; + } + + /** + * Rename: assert the outcome is true. + * @param src source path + * @param dest dest path + */ + private void assertRenamed(final Path src, final Path dest) + throws IOException { + assertTrue("rename(" + src + ", " + dest + ") failed", + getFileSystem().rename(src, dest)); + } + + /** + * Convert a path to a key; does not add any trailing / . + * @param path path in + * @return key out + */ + private String toKey(final Path path) { + return getFileSystem().pathToKey(path); + } + + /** + * Escape paths before handing to globStatus; this is needed as + * parameterized runs produce paths with [] in them. + * @param pathstr source path string + * @return an escaped path string + */ + private String escape(String pathstr) { + StringBuilder r = new StringBuilder(); + for (char c : pathstr.toCharArray()) { + String ch = Character.toString(c); + if ("?*[{".contains(ch)) { + r.append("\\"); + } + r.append(ch); + } + return r.toString(); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java new file mode 100644 index 0000000000000..d3d976e928940 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + + +import java.util.Arrays; +import java.util.Collection; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.Tristate; +import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; + +import static org.apache.hadoop.fs.s3a.Statistic.*; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; +import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe; + +/** + * Use metrics to assert about the cost of file API calls. + *

+ * Parameterized on guarded vs raw. and directory marker keep vs delete. + */ +@RunWith(Parameterized.class) +public class ITestS3ADeleteCost extends AbstractS3ACostTest { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestS3ADeleteCost.class); + + /** + * Parameterization. + */ + @Parameterized.Parameters(name = "{0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {"raw-keep-markers", false, true, false}, + {"raw-delete-markers", false, false, false}, + {"nonauth-keep-markers", true, true, false}, + {"auth-delete-markers", true, false, true} + }); + } + + public ITestS3ADeleteCost(final String name, + final boolean s3guard, + final boolean keepMarkers, + final boolean authoritative) { + super(s3guard, keepMarkers, authoritative); + } + + @Override + public void teardown() throws Exception { + if (isKeepingMarkers()) { + // do this ourselves to avoid audits teardown failing + // when surplus markers are found + deleteTestDirInTeardown(); + } + super.teardown(); + } + + /** + * This creates a directory with a child and then deletes it. + * The parent dir must be found and declared as empty. + *

When deleting markers, that forces the recreation of a new marker.

+ */ + @Test + public void testDeleteSingleFileInDir() throws Throwable { + describe("delete a file"); + S3AFileSystem fs = getFileSystem(); + // creates the marker + Path dir = dir(methodPath()); + // file creation may have deleted that marker, but it may + // still be there + Path simpleFile = file(new Path(dir, "simple.txt")); + + boolean rawAndKeeping = isRaw() && isDeleting(); + boolean rawAndDeleting = isRaw() && isDeleting(); + verifyMetrics(() -> { + fs.delete(simpleFile, false); + return "after fs.delete(simpleFile) " + getMetricSummary(); + }, + probe(rawAndKeeping, OBJECT_METADATA_REQUESTS, + FILESTATUS_FILE_PROBE_H), + // if deleting markers, look for the parent too + probe(rawAndDeleting, OBJECT_METADATA_REQUESTS, + FILESTATUS_FILE_PROBE_H + FILESTATUS_DIR_PROBE_H), + withWhenRaw(OBJECT_LIST_REQUESTS, + FILESTATUS_FILE_PROBE_L + FILESTATUS_DIR_PROBE_L), + with(DIRECTORIES_DELETED, 0), + with(FILES_DELETED, 1), + + // keeping: create no parent dirs or delete parents + withWhenKeeping(DIRECTORIES_CREATED, 0), + withWhenKeeping(OBJECT_DELETE_REQUESTS, DELETE_OBJECT_REQUEST), + + // deleting: create a parent and delete any of its parents + withWhenDeleting(DIRECTORIES_CREATED, 1), + withWhenDeleting(OBJECT_DELETE_REQUESTS, + DELETE_OBJECT_REQUEST + + DELETE_MARKER_REQUEST) + ); + // there is an empty dir for a parent + S3AFileStatus status = verifyRawInnerGetFileStatus(dir, true, + StatusProbeEnum.ALL, GET_FILE_STATUS_ON_DIR); + assertEmptyDirStatus(status, Tristate.TRUE); + } + + /** + * This creates a directory with a two files and then deletes one of the + * files. + */ + @Test + public void testDeleteFileInDir() throws Throwable { + describe("delete a file in a directory with multiple files"); + S3AFileSystem fs = getFileSystem(); + // creates the marker + Path dir = dir(methodPath()); + // file creation may have deleted that marker, but it may + // still be there + Path file1 = file(new Path(dir, "file1.txt")); + Path file2 = file(new Path(dir, "file2.txt")); + + boolean rawAndKeeping = isRaw() && isDeleting(); + boolean rawAndDeleting = isRaw() && isDeleting(); + verifyMetrics(() -> { + fs.delete(file1, false); + return "after fs.delete(file1simpleFile) " + getMetricSummary(); + }, + // delete file. For keeping: that's it + probe(rawAndKeeping, OBJECT_METADATA_REQUESTS, + FILESTATUS_FILE_PROBE_H), + // if deleting markers, look for the parent too + probe(rawAndDeleting, OBJECT_METADATA_REQUESTS, + FILESTATUS_FILE_PROBE_H + FILESTATUS_DIR_PROBE_H), + withWhenRaw(OBJECT_LIST_REQUESTS, + FILESTATUS_FILE_PROBE_L + FILESTATUS_DIR_PROBE_L), + with(DIRECTORIES_DELETED, 0), + with(FILES_DELETED, 1), + + // no need to create a parent + with(DIRECTORIES_CREATED, 0), + + // keeping: create no parent dirs or delete parents + withWhenKeeping(OBJECT_DELETE_REQUESTS, DELETE_OBJECT_REQUEST), + + // deleting: create a parent and delete any of its parents + withWhenDeleting(OBJECT_DELETE_REQUESTS, + DELETE_OBJECT_REQUEST)); + } + + @Test + public void testDirMarkersSubdir() throws Throwable { + describe("verify cost of deep subdir creation"); + + Path subDir = new Path(methodPath(), "1/2/3/4/5/6"); + // one dir created, possibly a parent removed + verifyMetrics(() -> { + mkdirs(subDir); + return "after mkdir(subDir) " + getMetricSummary(); + }, + with(DIRECTORIES_CREATED, 1), + with(DIRECTORIES_DELETED, 0), + withWhenKeeping(OBJECT_DELETE_REQUESTS, 0), + withWhenKeeping(FAKE_DIRECTORIES_DELETED, 0), + withWhenDeleting(OBJECT_DELETE_REQUESTS, DELETE_MARKER_REQUEST), + // delete all possible fake dirs above the subdirectory + withWhenDeleting(FAKE_DIRECTORIES_DELETED, + directoriesInPath(subDir) - 1)); + } + + @Test + public void testDirMarkersFileCreation() throws Throwable { + describe("verify cost of file creation"); + + Path srcBaseDir = dir(methodPath()); + + Path srcDir = dir(new Path(srcBaseDir, "1/2/3/4/5/6")); + + // creating a file should trigger demise of the src dir marker + // unless markers are being kept + + verifyMetrics(() -> { + file(new Path(srcDir, "source.txt")); + return "after touch(fs, srcFilePath) " + getMetricSummary(); + }, + with(DIRECTORIES_CREATED, 0), + with(DIRECTORIES_DELETED, 0), + // keeping: no delete operations. + withWhenKeeping(OBJECT_DELETE_REQUESTS, 0), + withWhenKeeping(FAKE_DIRECTORIES_DELETED, 0), + // delete all possible fake dirs above the file + withWhenDeleting(OBJECT_DELETE_REQUESTS, 1), + withWhenDeleting(FAKE_DIRECTORIES_DELETED, + directoriesInPath(srcDir))); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java new file mode 100644 index 0000000000000..85c70768356e6 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + + +import java.util.Arrays; +import java.util.Collection; +import java.util.UUID; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AFileSystem; + +import static org.apache.hadoop.fs.s3a.Statistic.*; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; + +/** + * Use metrics to assert about the cost of file API calls. + *

+ * Parameterized on guarded vs raw. and directory marker keep vs delete + */ +@RunWith(Parameterized.class) +public class ITestS3ARenameCost extends AbstractS3ACostTest { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestS3ARenameCost.class); + + /** + * Parameterization. + */ + @Parameterized.Parameters(name = "{0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {"raw-keep-markers", false, true, false}, + {"raw-delete-markers", false, false, false}, + {"nonauth-keep-markers", true, true, false}, + {"auth-delete-markers", true, false, true} + }); + } + + public ITestS3ARenameCost(final String name, + final boolean s3guard, + final boolean keepMarkers, + final boolean authoritative) { + super(s3guard, keepMarkers, authoritative); + } + + @Test + public void testRenameFileToDifferentDirectory() throws Throwable { + describe("rename a file to a different directory, " + + "keeping the source dir present"); + + Path baseDir = dir(methodPath()); + + Path srcDir = new Path(baseDir, "1/2/3/4/5/6"); + final Path srcFilePath = file(new Path(srcDir, "source.txt")); + + // create a new source file. + // Explicitly use a new path object to guarantee that the parent paths + // are different object instances and so equals() rather than == + // is + Path parent2 = srcFilePath.getParent(); + Path srcFile2 = file(new Path(parent2, "source2.txt")); + Assertions.assertThat(srcDir) + .isNotSameAs(parent2); + Assertions.assertThat(srcFilePath.getParent()) + .isEqualTo(srcFile2.getParent()); + + // create a directory tree, expect the dir to be created and + // possibly a request to delete all parent directories made. + Path destBaseDir = new Path(baseDir, "dest"); + Path destDir = dir(new Path(destBaseDir, "a/b/c/d")); + Path destFilePath = new Path(destDir, "dest.txt"); + + // rename the source file to the destination file. + // this tests file rename, not dir rename + // as srcFile2 exists, the parent dir of srcFilePath must not be created. + verifyMetrics(() -> + execRename(srcFilePath, destFilePath), + whenRaw(RENAME_SINGLE_FILE_DIFFERENT_DIR), + with(DIRECTORIES_CREATED, 0), + with(DIRECTORIES_DELETED, 0), + // keeping: only the core delete operation is issued. + withWhenKeeping(OBJECT_DELETE_REQUESTS, DELETE_OBJECT_REQUEST), + withWhenKeeping(FAKE_DIRECTORIES_DELETED, 0), + // deleting: delete any fake marker above the destination. + withWhenDeleting(OBJECT_DELETE_REQUESTS, + DELETE_OBJECT_REQUEST + DELETE_MARKER_REQUEST), + withWhenDeleting(FAKE_DIRECTORIES_DELETED, + directoriesInPath(destDir))); + + assertIsFile(destFilePath); + assertIsDirectory(srcDir); + assertPathDoesNotExist("should have gone in the rename", srcFilePath); + } + + /** + * Same directory rename is lower cost as there's no need to + * look for the parent dir of the dest path or worry about + * deleting markers. + */ + @Test + public void testRenameSameDirectory() throws Throwable { + describe("rename a file to the same directory"); + + Path baseDir = dir(methodPath()); + final Path sourceFile = file(new Path(baseDir, "source.txt")); + + // create a new source file. + // Explicitly use a new path object to guarantee that the parent paths + // are different object instances and so equals() rather than == + // is + Path parent2 = sourceFile.getParent(); + Path destFile = new Path(parent2, "dest"); + verifyMetrics(() -> + execRename(sourceFile, destFile), + whenRaw(RENAME_SINGLE_FILE_SAME_DIR), + with(OBJECT_COPY_REQUESTS, 1), + with(DIRECTORIES_CREATED, 0), + with(OBJECT_DELETE_REQUESTS, DELETE_OBJECT_REQUEST), + with(FAKE_DIRECTORIES_DELETED, 0)); + } + + @Test + public void testCostOfRootFileRename() throws Throwable { + describe("assert that a root file rename doesn't" + + " do much in terms of parent dir operations"); + S3AFileSystem fs = getFileSystem(); + + // unique name, so that even when run in parallel tests, there's no conflict + String uuid = UUID.randomUUID().toString(); + Path src = file(new Path("/src-" + uuid)); + Path dest = new Path("/dest-" + uuid); + try { + verifyMetrics(() -> { + fs.rename(src, dest); + return "after fs.rename(/src,/dest) " + getMetricSummary(); + }, + whenRaw(FILE_STATUS_FILE_PROBE + .plus(GET_FILE_STATUS_FNFE) + .plus(COPY_OP)), + // here we expect there to be no fake directories + with(DIRECTORIES_CREATED, 0), + // one for the renamed file only + with(OBJECT_DELETE_REQUESTS, + DELETE_OBJECT_REQUEST), + // no directories are deleted: This is root + with(DIRECTORIES_DELETED, 0), + // no fake directories are deleted: This is root + with(FAKE_DIRECTORIES_DELETED, 0), + with(FILES_DELETED, 1)); + } finally { + fs.delete(src, false); + fs.delete(dest, false); + } + } + + @Test + public void testCostOfRootFileDelete() throws Throwable { + describe("assert that a root file delete doesn't" + + " do much in terms of parent dir operations"); + S3AFileSystem fs = getFileSystem(); + + // unique name, so that even when run in parallel tests, there's no conflict + String uuid = UUID.randomUUID().toString(); + Path src = file(new Path("/src-" + uuid)); + try { + // delete that destination file, assert only the file delete was issued + verifyMetrics(() -> { + fs.delete(src, false); + return "after fs.delete(/dest) " + getMetricSummary(); + }, + with(DIRECTORIES_CREATED, 0), + with(DIRECTORIES_DELETED, 0), + with(FAKE_DIRECTORIES_DELETED, 0), + with(FILES_DELETED, 1), + with(OBJECT_DELETE_REQUESTS, DELETE_OBJECT_REQUEST), + whenRaw(FILE_STATUS_FILE_PROBE)); /* no need to look at parent. */ + + } finally { + fs.delete(src, false); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java new file mode 100644 index 0000000000000..46a6b712c49bf --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + +/** + * Declaration of the costs of head and list calls for various FS IO + * operations. + *

+ * An instance declares the number of head and list calls expected for + * various operations -with a {@link #plus(OperationCost)} + * method to add operation costs together to produce an + * aggregate cost. These can then be validated in tests + * via {@link OperationCostValidator}. + * + */ +public final class OperationCost { + + /** Head costs for getFileStatus() directory probe: {@value}. */ + public static final int FILESTATUS_DIR_PROBE_H = 0; + + /** List costs for getFileStatus() directory probe: {@value}. */ + public static final int FILESTATUS_DIR_PROBE_L = 1; + + /** Head cost getFileStatus() file probe only. */ + public static final int FILESTATUS_FILE_PROBE_H = 1; + + /** Liast cost getFileStatus() file probe only. */ + + public static final int FILESTATUS_FILE_PROBE_L = 0; + + /** + * Delete cost when deleting an object. + */ + public static final int DELETE_OBJECT_REQUEST = 1; + + /** + * Delete cost when deleting a marker. + */ + public static final int DELETE_MARKER_REQUEST = DELETE_OBJECT_REQUEST; + + /** + * No IO takes place. + */ + public static final OperationCost NO_IO = + new OperationCost(0, 0); + + /** A HEAD operation. */ + public static final OperationCost HEAD_OPERATION = new OperationCost(1, 0); + + /** A LIST operation. */ + public static final OperationCost LIST_OPERATION = new OperationCost(0, 1); + + /** + * Cost of {@link org.apache.hadoop.fs.s3a.impl.StatusProbeEnum#DIRECTORIES}. + */ + public static final OperationCost FILE_STATUS_DIR_PROBE = LIST_OPERATION; + + /** + * Cost of {@link org.apache.hadoop.fs.s3a.impl.StatusProbeEnum#FILE}. + */ + public static final OperationCost FILE_STATUS_FILE_PROBE = HEAD_OPERATION; + + /** + * Cost of {@link org.apache.hadoop.fs.s3a.impl.StatusProbeEnum#ALL}. + */ + public static final OperationCost FILE_STATUS_ALL_PROBES = + FILE_STATUS_FILE_PROBE.plus(FILE_STATUS_DIR_PROBE); + + /** getFileStatus() on a file which exists. */ + public static final OperationCost GET_FILE_STATUS_ON_FILE = + FILE_STATUS_FILE_PROBE; + + /** List costs for getFileStatus() on a non-empty directory: {@value}. */ + public static final OperationCost GET_FILE_STATUS_ON_DIR = + FILE_STATUS_FILE_PROBE.plus(FILE_STATUS_DIR_PROBE); + + /** Costs for getFileStatus() on an empty directory: {@value}. */ + public static final OperationCost GET_FILE_STATUS_ON_EMPTY_DIR = + GET_FILE_STATUS_ON_DIR; + + /** getFileStatus() directory marker which exists. */ + public static final OperationCost GET_FILE_STATUS_ON_DIR_MARKER = + GET_FILE_STATUS_ON_EMPTY_DIR; + + /** getFileStatus() call which fails to find any entry. */ + public static final OperationCost GET_FILE_STATUS_FNFE = + FILE_STATUS_ALL_PROBES; + + /** listLocatedStatus always does a LIST. */ + public static final OperationCost LIST_LOCATED_STATUS_LIST_OP = + new OperationCost(0, 1); + + /** listFiles always does a LIST. */ + public static final OperationCost LIST_FILES_LIST_OP = + new OperationCost(0, 1); + + /** + * Metadata cost of a copy operation, as used during rename. + * This happens even if the store is guarded. + */ + public static final OperationCost COPY_OP = + new OperationCost(1, 0); + + /** + * Cost of renaming a file to a different directory. + *

+ * LIST on dest not found, look for dest dir, and then, at + * end of rename, whether a parent dir needs to be created. + */ + public static final OperationCost RENAME_SINGLE_FILE_DIFFERENT_DIR = + FILE_STATUS_FILE_PROBE // source file probe + .plus(GET_FILE_STATUS_FNFE) // dest does not exist + .plus(FILE_STATUS_DIR_PROBE) // parent dir of dest + .plus(FILE_STATUS_DIR_PROBE) // recreate source parent dir? + .plus(COPY_OP); // metadata read on copy + + /** + * Cost of renaming a file to the same directory + *

+ * No need to look for parent directories, so only file + * existence checks and the copy. + */ + public static final OperationCost RENAME_SINGLE_FILE_SAME_DIR = + FILE_STATUS_FILE_PROBE // source file probe + .plus(GET_FILE_STATUS_FNFE) // dest must not exist + .plus(COPY_OP); // metadata read on copy + + /** + * create(overwrite = true) does not look for the file existing. + */ + public static final OperationCost CREATE_FILE_OVERWRITE = + FILE_STATUS_DIR_PROBE; + + /** + * create(overwrite = false) runs all the checks. + */ + public static final OperationCost CREATE_FILE_NO_OVERWRITE = + FILE_STATUS_ALL_PROBES; + + /** Expected HEAD count. */ + private final int head; + + /** Expected LIST count. */ + private final int list; + + /** + * Constructor. + * @param head head requests. + * @param list list requests. + */ + public OperationCost(final int head, + final int list) { + this.head = head; + this.list = list; + } + + /** Expected HEAD count. */ + int head() { + return head; + } + + /** Expected LIST count. */ + int list() { + return list; + } + + /** + * Add to create a new cost. + * @param that the other entry + * @return cost of the combined operation. + */ + public OperationCost plus(OperationCost that) { + return new OperationCost( + head + that.head, + list + that.list); + } + + @Override + public String toString() { + return "OperationCost{" + + "head=" + head + + ", list=" + list + + '}'; + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java new file mode 100644 index 0000000000000..c351d1b185a32 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java @@ -0,0 +1,483 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +import org.assertj.core.api.Assumptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.apache.hadoop.fs.s3a.Statistic; + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_LIST_REQUESTS; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_METADATA_REQUESTS; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Support for declarative assertions about operation cost. + *

+ * Usage: A builder is used to declare the set of statistics + * to be monitored in the filesystem. + *

+ * A call to {@link #exec(Callable, ExpectedProbe...)} + * executes the callable if 1+ probe is enabled; after + * invocation the probes are validated. + * The result of the callable is returned. + *

+ * A call of {@link #intercepting(Class, String, Callable, ExpectedProbe...)} + * Invokes the callable if 1+ probe is enabled, expects an exception + * to be raised and then verifies metrics declared in the probes. + *

+ * Probes are built up from the static method to create probes + * for metrics: + *
    + *
  • {@link #probe(boolean, Statistic, int)}
    • + *
  • {@link #probe(Statistic, int)}
    • + *
  • {@link #probes(boolean, ExpectedProbe...)} (Statistic, int)}
    • + *
  • {@link #always()}
    • + *
+ * If any probe evaluates to false, an assertion is raised. + *

+ * When this happens: look in the logs! + * The logs will contain the whole set of metrics, the probe details + * and the result of the call. + */ +public final class OperationCostValidator { + + private static final Logger LOG = + LoggerFactory.getLogger(OperationCostValidator.class); + + /** + * The empty probe: declared as disabled. + */ + private static final ExpectedProbe EMPTY_PROBE = + new EmptyProbe("empty", false); + + /** + * A probe which is always enabled. + */ + private static final ExpectedProbe ALWAYS_PROBE = + new EmptyProbe("always", true); + + /** + * The map of metric diffs to track. + */ + private final Map metricDiffs + = new TreeMap<>(); + + /** + * Build the instance. + * @param builder builder containing all options. + */ + private OperationCostValidator(Builder builder) { + builder.metrics.forEach(stat -> + metricDiffs.put(stat.getSymbol(), + new S3ATestUtils.MetricDiff(builder.filesystem, stat))); + builder.metrics.clear(); + } + + /** + * Reset all the metrics being tracked. + */ + public void resetMetricDiffs() { + metricDiffs.values().forEach(S3ATestUtils.MetricDiff::reset); + } + + /** + * Get the diff of a statistic. + * @param stat statistic to look up + * @return the value + * @throws NullPointerException if there is no match + */ + public S3ATestUtils.MetricDiff get(Statistic stat) { + S3ATestUtils.MetricDiff diff = + requireNonNull(metricDiffs.get(stat.getSymbol()), + () -> "No metric tracking for " + stat); + return diff; + } + + /** + * Execute a closure and verify the metrics. + *

+ * If no probes are active, the operation will + * raise an Assumption exception for the test to be skipped. + * @param eval closure to evaluate + * @param expected varargs list of expected diffs + * @param return type. + * @return the result of the evaluation + */ + public T exec( + Callable eval, + ExpectedProbe... expectedA) throws Exception { + List expected = Arrays.asList(expectedA); + resetMetricDiffs(); + // verify that 1+ probe is enabled + assumeProbesEnabled(expected); + // if we get here, then yes. + // evaluate it + T r = eval.call(); + // build the text for errors + String text = + "operation returning " + + (r != null ? r.toString() : "null"); + LOG.info("{}", text); + LOG.info("state {}", this); + LOG.info("probes {}", expected); + for (ExpectedProbe ed : expected) { + ed.verify(this, text); + } + return r; + } + + /** + * Scan all probes for being enabled. + *

+ * If none of them are enabled, the evaluation will be skipped. + * @param expected list of expected probes + */ + private void assumeProbesEnabled(List expected) { + boolean enabled = false; + for (ExpectedProbe ed : expected) { + enabled |= ed.isEnabled(); + } + String pstr = expected.stream() + .map(Object::toString) + .collect(Collectors.joining(", ")); + Assumptions.assumeThat(enabled) + .describedAs("metrics to probe for are not enabled in %s", pstr) + .isTrue(); + } + + /** + * Execute a closure, expecting an exception. + * Verify the metrics after the exception has been caught and + * validated. + * @param clazz type of exception + * @param text text to look for in exception (optional) + * @param eval closure to evaluate + * @param expected varargs list of expected diffs + * @param return type of closure + * @param exception type + * @return the exception caught. + * @throws Exception any other exception + */ + public E intercepting( + Class clazz, + String text, + Callable eval, + ExpectedProbe... expected) throws Exception { + + return exec(() -> + intercept(clazz, text, eval), + expected); + } + + @Override + public String toString() { + return metricDiffs.values().stream() + .map(S3ATestUtils.MetricDiff::toString) + .collect(Collectors.joining(", ")); + } + + /** + * Create a builder for the cost checker. + * + * @param fs filesystem. + * @return builder. + */ + public static Builder builder(S3AFileSystem fs) { + return new Builder(fs); + } + + /** + * builder. + */ + public static final class Builder { + + /** + * Filesystem. + */ + private final S3AFileSystem filesystem; + + /** + * Metrics to create. + */ + private final List metrics = new ArrayList<>(); + + + /** + * Create with a required filesystem. + * @param filesystem monitored filesystem + */ + public Builder(final S3AFileSystem filesystem) { + this.filesystem = requireNonNull(filesystem); + } + + + /** + * Add a single metric. + * @param statistic statistic to monitor. + * @return this + */ + public Builder withMetric(Statistic statistic) { + return withMetric(statistic); + } + + /** + * Add a varargs list of metrics. + * @param stat statistics to monitor. + * @return this. + */ + public Builder withMetrics(Statistic...stats) { + metrics.addAll(Arrays.asList(stats)); + return this; + } + + /** + * Instantiate. + * @return the validator. + */ + public OperationCostValidator build() { + return new OperationCostValidator(this); + } + } + + /** + * Get the "always" probe. + * @return a probe which always triggers execution. + */ + public static ExpectedProbe always() { + return ALWAYS_PROBE; + } + + /** + * Create a probe of a statistic which is enabled whenever the expected + * value is greater than zero. + * @param statistic statistic to check. + * @param expected expected value. + * @return a probe. + */ + public static ExpectedProbe probe( + final Statistic statistic, + final int expected) { + return probe(expected >= 0, statistic, expected); + } + + /** + * Create a probe of a statistic which is conditionally enabled. + * @param enabled is the probe enabled? + * @param statistic statistic to check. + * @param expected expected value. + * @return a probe. + */ + public static ExpectedProbe probe( + final boolean enabled, + final Statistic statistic, + final int expected) { + return enabled + ? new ExpectSingleStatistic(statistic, expected) + : EMPTY_PROBE; + } + + /** + * Create an aggregate probe from a vararges list of probes. + * @param enabled should the probes be enabled? + * @param plist probe list + * @return a probe + */ + public static ExpectedProbe probes( + final boolean enabled, + final ExpectedProbe...plist) { + return enabled + ? new ProbeList(Arrays.asList(plist)) + : EMPTY_PROBE; + } + + /** + * Expect the exact head and list requests of the operation + * cost supplied. + * @param enabled is the probe enabled? + * @param cost expected cost. + * @return a probe. + */ + public static ExpectedProbe expect( + boolean enabled, OperationCost cost) { + return probes(enabled, + probe(OBJECT_METADATA_REQUESTS, cost.head()), + probe(OBJECT_LIST_REQUESTS, cost.list())); + } + + /** + * An expected probe to verify given criteria to trigger an eval. + *

+ * Probes can be conditional, in which case they are only evaluated + * when true. + */ + public interface ExpectedProbe { + + /** + * Verify a diff if the FS instance is compatible. + * @param message message to print; metric name is appended + */ + void verify(OperationCostValidator diffs, String message); + + boolean isEnabled(); + } + + /** + * Simple probe is a single statistic. + */ + public static final class ExpectSingleStatistic implements ExpectedProbe { + + private final Statistic statistic; + + private final int expected; + + /** + * Create. + * @param statistic statistic + * @param expected expected value. + */ + private ExpectSingleStatistic(final Statistic statistic, + final int expected) { + this.statistic = statistic; + this.expected = expected; + } + + /** + * Verify a diff if the FS instance is compatible. + * @param message message to print; metric name is appended + */ + @Override + public void verify(OperationCostValidator diffs, String message) { + diffs.get(statistic).assertDiffEquals(message, expected); + } + + public Statistic getStatistic() { + return statistic; + } + + public int getExpected() { + return expected; + } + + @Override + public boolean isEnabled() { + return true; + } + + @Override + public String toString() { + String sb = "ExpectSingleStatistic{" + + statistic + + ", expected=" + expected + + ", enabled=" + isEnabled() + + '}'; + return sb; + } + } + + /** + * A list of probes; the verify operation + * verifies them all. + */ + public static class ProbeList implements ExpectedProbe { + + /** + * Probe list. + */ + private final List probes; + + /** + * Constructor. + * @param probes probe list. + */ + public ProbeList(final List probes) { + this.probes = probes; + } + + @Override + public void verify(final OperationCostValidator diffs, + final String message) { + probes.forEach(p -> p.verify(diffs, message)); + } + + /** + * Enabled if 1+ probe is enabled. + * @return true if enabled. + */ + @Override + public boolean isEnabled() { + boolean enabled = false; + for (ExpectedProbe probe : probes) { + enabled |= probe.isEnabled(); + } + return enabled; + } + + @Override + public String toString() { + String pstr = probes.stream() + .map(Object::toString) + .collect(Collectors.joining(", ")); + return "ProbeList{" + pstr + '}'; + } + } + + /** + * The empty probe always runs; it can be used to force + * a verification to execute. + */ + private static final class EmptyProbe implements ExpectedProbe { + + private final String name; + + private final boolean enabled; + + private EmptyProbe(final String name, boolean enabled) { + this.name = name; + this.enabled = enabled; + } + + @Override + public void verify(final OperationCostValidator diffs, + final String message) { + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public String toString() { + return name; + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index 3e187a1515630..64057d02f8220 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -60,10 +60,14 @@ import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL; import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL; import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.BucketInfo.IS_MARKER_AWARE; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.INVALID_ARGUMENT; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.runS3GuardCommand; +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.MARKERS; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_ACCEPTABLE; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -124,7 +128,7 @@ protected static void expectResult(int expected, public static String expectSuccess( String message, S3GuardTool tool, - String... args) throws Exception { + Object... args) throws Exception { ByteArrayOutputStream buf = new ByteArrayOutputStream(); exec(SUCCESS, message, tool, buf, args); return buf.toString(); @@ -137,9 +141,9 @@ public static String expectSuccess( * @return the return code * @throws Exception any exception */ - protected int run(Configuration conf, String... args) + protected int run(Configuration conf, Object... args) throws Exception { - return S3GuardTool.run(conf, args); + return runS3GuardCommand(conf, args); } /** @@ -149,8 +153,8 @@ protected int run(Configuration conf, String... args) * @return the return code * @throws Exception any exception */ - protected int run(String... args) throws Exception { - return S3GuardTool.run(getConfiguration(), args); + protected int run(Object... args) throws Exception { + return runS3GuardCommand(getConfiguration(), args); } /** @@ -160,11 +164,12 @@ protected int run(String... args) throws Exception { * @param args argument list * @throws Exception any exception */ - protected void runToFailure(int status, String... args) + protected void runToFailure(int status, Object... args) throws Exception { + final Configuration conf = getConfiguration(); ExitUtil.ExitException ex = - intercept(ExitUtil.ExitException.class, - () -> run(args)); + intercept(ExitUtil.ExitException.class, () -> + runS3GuardCommand(conf, args)); if (ex.status != status) { throw ex; } @@ -445,6 +450,44 @@ public void testBucketInfoUnguarded() throws Exception { info.contains("S3A Client")); } + /** + * Verify that the {@code -markers aware} option works. + * This test case is in this class for ease of backporting. + */ + @Test + public void testBucketInfoMarkerAware() throws Throwable { + final Configuration conf = getConfiguration(); + URI fsUri = getFileSystem().getUri(); + + // run a bucket info command and look for + // confirmation that it got the output from DDB diags + S3GuardTool.BucketInfo infocmd = toClose(new S3GuardTool.BucketInfo(conf)); + String info = exec(infocmd, S3GuardTool.BucketInfo.NAME, + "-" + MARKERS, S3GuardTool.BucketInfo.MARKERS_AWARE, + fsUri.toString()); + + assertTrue("Output should contain information about S3A client " + info, + info.contains(IS_MARKER_AWARE)); + } + + /** + * Verify that the {@code -markers} option fails on unknown options. + * This test case is in this class for ease of backporting. + */ + @Test + public void testBucketInfoMarkerPolicyUnknown() throws Throwable { + final Configuration conf = getConfiguration(); + URI fsUri = getFileSystem().getUri(); + + // run a bucket info command and look for + // confirmation that it got the output from DDB diags + S3GuardTool.BucketInfo infocmd = toClose(new S3GuardTool.BucketInfo(conf)); + intercept(ExitUtil.ExitException.class, ""+ EXIT_NOT_ACCEPTABLE, () -> + exec(infocmd, S3GuardTool.BucketInfo.NAME, + "-" + MARKERS, "unknown", + fsUri.toString())); + } + @Test public void testSetCapacityFailFastIfNotGuarded() throws Exception{ Configuration conf = getConfiguration(); @@ -654,4 +697,5 @@ public void testInitFailsIfNoBucketNameOrDDBTableSet() throws Exception { assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3); assertFalse("Diff contained duplicates", duplicates); } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardDDBRootOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardDDBRootOperations.java index b2e6b3e93a8b3..afb0fd8c55a7b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardDDBRootOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardDDBRootOperations.java @@ -38,9 +38,12 @@ import org.apache.hadoop.fs.s3a.impl.StoreContext; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY; import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides; import static org.apache.hadoop.fs.s3a.S3AUtils.applyLocatedFiles; @@ -52,6 +55,8 @@ * integration tests. *

* The tests only run if DynamoDB is the metastore. + *

+ * The marker policy is fixed to "delete" */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class ITestS3GuardDDBRootOperations extends AbstractS3ATestBase { @@ -82,9 +87,15 @@ protected int getTestTimeoutMillis() { protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); String bucketName = getTestBucketName(conf); + disableFilesystemCaching(conf); + removeBucketOverrides(bucketName, conf, + S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY, + ENABLE_MULTI_DELETE, + DIRECTORY_MARKER_POLICY); + conf.set(DIRECTORY_MARKER_POLICY, + DIRECTORY_MARKER_POLICY_DELETE); // set a sleep time of 0 on pruning, for speedier test runs. - removeBucketOverrides(bucketName, conf, ENABLE_MULTI_DELETE); conf.setTimeDuration( S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY, 0, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestHelper.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestHelper.java index 4a5e55eb61e3c..89b4051de8776 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestHelper.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestHelper.java @@ -20,12 +20,16 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ExitCodeProvider; +import org.apache.hadoop.util.ExitUtil; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assert.assertEquals; /** @@ -48,7 +52,7 @@ private S3GuardToolTestHelper() { * @param args argument list * @throws Exception on any failure */ - public static String exec(S3GuardTool cmd, String... args) throws Exception { + public static String exec(S3GuardTool cmd, Object... args) throws Exception { return expectExecResult(0, cmd, args); } @@ -64,7 +68,7 @@ public static String exec(S3GuardTool cmd, String... args) throws Exception { public static String expectExecResult( final int expectedResult, final S3GuardTool cmd, - final String... args) throws Exception { + final Object... args) throws Exception { ByteArrayOutputStream buf = new ByteArrayOutputStream(); try { exec(expectedResult, "", cmd, buf, args); @@ -77,6 +81,17 @@ public static String expectExecResult( } } + /** + * Given an array of objects, conver to an array of strings. + * @param oargs object args + * @return string equivalent + */ + public static String[] varargsToString(final Object[] oargs) { + return Arrays.stream(oargs) + .map(Object::toString) + .toArray(String[]::new); + } + /** * Execute a command, saving the output into the buffer. * @param expectedResult expected result of the command. @@ -91,8 +106,9 @@ public static void exec(final int expectedResult, final String errorText, final S3GuardTool cmd, final ByteArrayOutputStream buf, - final String... args) + final Object... oargs) throws Exception { + final String[] args = varargsToString(oargs); LOG.info("exec {}", (Object) args); int r; try (PrintStream out = new PrintStream(buf)) { @@ -116,4 +132,43 @@ public static void exec(final int expectedResult, } } + /** + * Run a S3GuardTool command from a varags list. + *

+ * Warning: if the filesystem is retrieved from the cache, + * it will be closed afterwards. + * @param conf configuration + * @param args argument list + * @return the return code + * @throws Exception any exception + */ + public static int runS3GuardCommand(Configuration conf, Object... args) + throws Exception { + return S3GuardTool.run(conf, varargsToString(args)); + } + + /** + * Run a S3GuardTool command from a varags list, catch any raised + * ExitException and verify the status code matches that expected. + * @param conf configuration + * @param status expected status code of the exception + * @param args argument list + * @throws Exception any exception + */ + public static void runS3GuardCommandToFailure(Configuration conf, + int status, + Object... args) throws Exception { + + ExitUtil.ExitException ex = + intercept(ExitUtil.ExitException.class, + () -> { + int ec = runS3GuardCommand(conf, args); + if (ec != 0) { + throw new ExitUtil.ExitException(ec, "exit code " + ec); + } + }); + if (ex.status != status) { + throw ex; + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java index 1ce3ee56ce0a5..5f7a6fbd072df 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java @@ -316,18 +316,23 @@ public void testRemoveExpiredEntriesFromListing() { List listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3); DirListingMetadata meta = new DirListingMetadata(path, listing, false); - meta.removeExpiredEntriesFromListing(ttl, now); + List expired = meta.removeExpiredEntriesFromListing(ttl, + now); Assertions.assertThat(meta.getListing()) .describedAs("Metadata listing for %s", path) .doesNotContain(pathMeta1) .contains(pathMeta2) .contains(pathMeta3); + Assertions.assertThat(expired) + .describedAs("Expire entries underr %s", path) + .doesNotContain(pathMeta2) + .contains(pathMeta1); } - /* + /** * Create DirListingMetadata with two dirs and one file living in directory - * 'parent' + * 'parent'. */ private static DirListingMetadata makeTwoDirsOneFile(Path parent) { PathMetadata pathMeta1 = new PathMetadata( diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java new file mode 100644 index 0000000000000..00e62d9491070 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.tools; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.StringUtils; + +import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.VERBOSE; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.runS3GuardCommand; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.runS3GuardCommandToFailure; +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.UNLIMITED_LISTING; + +/** + * Class for marker tool tests -sets up keeping/deleting filesystems, + * has methods to invoke. + */ +public class AbstractMarkerToolTest extends AbstractS3ATestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(AbstractMarkerToolTest.class); + + /** the -verbose option. */ + protected static final String V = AbstractMarkerToolTest.m(VERBOSE); + + /** FS which keeps markers. */ + private S3AFileSystem keepingFS; + + /** FS which deletes markers. */ + private S3AFileSystem deletingFS; + + /** FS which mixes markers; only created in some tests. */ + private S3AFileSystem mixedFS; + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + String bucketName = getTestBucketName(conf); + removeBaseAndBucketOverrides(bucketName, conf, + S3A_BUCKET_PROBE, + DIRECTORY_MARKER_POLICY, + S3_METADATA_STORE_IMPL, + METADATASTORE_AUTHORITATIVE, + AUTHORITATIVE_PATH); + // base FS is legacy + conf.set(DIRECTORY_MARKER_POLICY, DIRECTORY_MARKER_POLICY_DELETE); + conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL); + + // turn off bucket probes for a bit of speedup in the connectors we create. + conf.setInt(S3A_BUCKET_PROBE, 0); + return conf; + } + + @Override + public void setup() throws Exception { + super.setup(); + setKeepingFS(createFS(DIRECTORY_MARKER_POLICY_KEEP, null)); + setDeletingFS(createFS(DIRECTORY_MARKER_POLICY_DELETE, null)); + } + + @Override + public void teardown() throws Exception { + // do this ourselves to avoid audits teardown failing + // when surplus markers are found + deleteTestDirInTeardown(); + super.teardown(); + IOUtils.cleanupWithLogger(LOG, getKeepingFS(), + getMixedFS(), getDeletingFS()); + + } + + /** + * FS which deletes markers. + */ + public S3AFileSystem getDeletingFS() { + return deletingFS; + } + + public void setDeletingFS(final S3AFileSystem deletingFS) { + this.deletingFS = deletingFS; + } + + /** + * FS which keeps markers. + */ + protected S3AFileSystem getKeepingFS() { + return keepingFS; + } + + private void setKeepingFS(S3AFileSystem keepingFS) { + this.keepingFS = keepingFS; + } + + /** only created on demand. */ + private S3AFileSystem getMixedFS() { + return mixedFS; + } + + protected void setMixedFS(S3AFileSystem mixedFS) { + this.mixedFS = mixedFS; + } + + /** + * Get a filename for a temp file. + * The generated file is deleted. + * + * @return a file path for a output file + */ + protected File tempAuditFile() throws IOException { + final File audit = File.createTempFile("audit", ".txt"); + audit.delete(); + return audit; + } + + /** + * Read the audit output and verify it has the expected number of lines. + * @param auditFile audit file to read + * @param expected expected line count + */ + protected void expectMarkersInOutput(final File auditFile, + final int expected) + throws IOException { + final List lines = readOutput(auditFile); + Assertions.assertThat(lines) + .describedAs("Content of %s", auditFile) + .hasSize(expected); + } + + /** + * Read the output file in. Logs the contents at info. + * @param outputFile audit output file. + * @return the lines + */ + protected List readOutput(final File outputFile) + throws IOException { + try (FileReader reader = new FileReader(outputFile)) { + final List lines = + org.apache.commons.io.IOUtils.readLines(reader); + + LOG.info("contents of output file {}\n{}", outputFile, + StringUtils.join("\n", lines)); + return lines; + } + } + + /** + * Create a new FS with given marker policy and path. + * This filesystem MUST be closed in test teardown. + * @param markerPolicy markers + * @param authPath authoritative path. If null: no path. + * @return a new FS. + */ + protected S3AFileSystem createFS(String markerPolicy, + String authPath) throws Exception { + S3AFileSystem testFS = getFileSystem(); + Configuration conf = new Configuration(testFS.getConf()); + URI testFSUri = testFS.getUri(); + String bucketName = getTestBucketName(conf); + removeBucketOverrides(bucketName, conf, + DIRECTORY_MARKER_POLICY, + S3_METADATA_STORE_IMPL, + BULK_DELETE_PAGE_SIZE, + AUTHORITATIVE_PATH); + if (authPath != null) { + conf.set(AUTHORITATIVE_PATH, authPath); + } + // Use a very small page size to force the paging + // code to be tested. + conf.setInt(BULK_DELETE_PAGE_SIZE, 2); + conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL); + conf.set(DIRECTORY_MARKER_POLICY, markerPolicy); + S3AFileSystem fs2 = new S3AFileSystem(); + fs2.initialize(testFSUri, conf); + LOG.info("created new filesystem with policy {} and auth path {}", + markerPolicy, + (authPath == null ? "(null)": authPath)); + return fs2; + } + + /** + * Execute the marker tool, expecting the execution to succeed. + * @param sourceFS filesystem to use + * @param path path to scan + * @param doPurge should markers be purged + * @param expectedMarkerCount number of markers expected + * @return the result + */ + protected MarkerTool.ScanResult markerTool( + final FileSystem sourceFS, + final Path path, + final boolean doPurge, + final int expectedMarkerCount) + throws IOException { + return markerTool(0, sourceFS, path, doPurge, + expectedMarkerCount, + UNLIMITED_LISTING, false); + } + + /** + * Run a S3GuardTool command from a varags list and the + * configuration returned by {@code getConfiguration()}. + * @param args argument list + * @return the return code + * @throws Exception any exception + */ + protected int run(Object... args) throws Exception { + return runS3GuardCommand(uncachedFSConfig(getConfiguration()), args); + } + + /** + * Take a configuration, copy it and disable FS Caching on + * the new one. + * @param conf source config + * @return a new, patched, config + */ + protected Configuration uncachedFSConfig(final Configuration conf) { + Configuration c = new Configuration(conf); + disableFilesystemCaching(c); + return c; + } + + /** + * given an FS instance, create a matching configuration where caching + * is disabled. + * @param fs source + * @return new config. + */ + protected Configuration uncachedFSConfig(final FileSystem fs) { + return uncachedFSConfig(fs.getConf()); + } + + /** + * Run a S3GuardTool command from a varags list, catch any raised + * ExitException and verify the status code matches that expected. + * @param status expected status code of the exception + * @param args argument list + * @throws Exception any exception + */ + protected void runToFailure(int status, Object... args) + throws Exception { + Configuration conf = uncachedFSConfig(getConfiguration()); + runS3GuardCommandToFailure(conf, status, args); + } + + /** + * Given a base and a filename, create a new path. + * @param base base path + * @param name name: may be empty, in which case the base path is returned + * @return a path + */ + protected static Path toPath(final Path base, final String name) { + return name.isEmpty() ? base : new Path(base, name); + } + + /** + * Execute the marker tool, expecting the execution to + * return a specific exit code. + * + * @param sourceFS filesystem to use + * @param exitCode exit code to expect. + * @param path path to scan + * @param doPurge should markers be purged + * @param expectedMarkers number of markers expected + * @param limit limit of files to scan; -1 for 'unlimited' + * @param nonAuth only use nonauth path count for failure rules + * @return the result + */ + public static MarkerTool.ScanResult markerTool( + final int exitCode, + final FileSystem sourceFS, + final Path path, + final boolean doPurge, + final int expectedMarkers, + final int limit, + final boolean nonAuth) throws IOException { + + MarkerTool.ScanResult result = MarkerTool.execMarkerTool( + sourceFS, + path, + doPurge, + expectedMarkers, + limit, nonAuth); + Assertions.assertThat(result.getExitCode()) + .describedAs("Exit code of marker(%s, %s, %d) -> %s", + path, doPurge, expectedMarkers, result) + .isEqualTo(exitCode); + return result; + } + + /** + * Add a "-" prefix to a string. + * @param s string to prefix + * @return a string for passing into the CLI + */ + protected static String m(String s) { + return "-" + s; + } + + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java new file mode 100644 index 0000000000000..4a81b1aba919b --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java @@ -0,0 +1,533 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.tools; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.S3AFileSystem; + +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_AUTHORITATIVE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.BucketInfo.BUCKET_INFO; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.runS3GuardCommand; +import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.runS3GuardCommandToFailure; +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.*; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_INTERRUPTED; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_ACCEPTABLE; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_FOUND; +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE; + +/** + * Test the marker tool and use it to compare the behavior + * of keeping vs legacy S3A FS instances. + */ +public class ITestMarkerTool extends AbstractMarkerToolTest { + + protected static final Logger LOG = + LoggerFactory.getLogger(ITestMarkerTool.class); + + /** + * How many files to expect. + */ + private int expectedFileCount; + + /** + * How many markers to expect under dir1. + */ + private int expectedMarkersUnderDir1; + + /** + * How many markers to expect under dir2. + */ + private int expectedMarkersUnderDir2; + + /** + * How many markers to expect across both dirs? + */ + private int expectedMarkers; + + /** + * How many markers to expect including the base directory? + */ + private int expectedMarkersWithBaseDir; + + + @Test + public void testCleanMarkersLegacyDir() throws Throwable { + describe("Clean markers under a deleting FS -expect none"); + CreatedPaths createdPaths = createPaths(getDeletingFS(), methodPath()); + markerTool(getDeletingFS(), createdPaths.base, false, 0); + markerTool(getDeletingFS(), createdPaths.base, true, 0); + } + + @Test + public void testCleanMarkersFileLimit() throws Throwable { + describe("Clean markers under a keeping FS -with file limit"); + CreatedPaths createdPaths = createPaths(getKeepingFS(), methodPath()); + + // audit will be interrupted + markerTool(EXIT_INTERRUPTED, getDeletingFS(), + createdPaths.base, false, 0, 1, false); + } + + @Test + public void testCleanMarkersKeepingDir() throws Throwable { + describe("Audit then clean markers under a deleting FS " + + "-expect markers to be found and then cleaned up"); + CreatedPaths createdPaths = createPaths(getKeepingFS(), methodPath()); + + // audit will find the expected entries + int expectedMarkerCount = createdPaths.dirs.size(); + S3AFileSystem fs = getDeletingFS(); + LOG.info("Auditing a directory with retained markers -expect failure"); + markerTool(EXIT_NOT_ACCEPTABLE, fs, + createdPaths.base, false, 0, UNLIMITED_LISTING, false); + + LOG.info("Auditing a directory expecting retained markers"); + markerTool(fs, createdPaths.base, false, + expectedMarkerCount); + + // we require that a purge didn't take place, so run the + // audit again. + LOG.info("Auditing a directory expecting retained markers"); + markerTool(fs, createdPaths.base, false, + expectedMarkerCount); + + LOG.info("Purging a directory of retained markers"); + // purge cleans up + assertMarkersDeleted(expectedMarkerCount, + markerTool(fs, createdPaths.base, true, expectedMarkerCount)); + // and a rerun doesn't find markers + LOG.info("Auditing a directory with retained markers -expect success"); + assertMarkersDeleted(0, + markerTool(fs, createdPaths.base, true, 0)); + } + + @Test + public void testRenameKeepingFS() throws Throwable { + describe("Rename with the keeping FS -verify that no markers" + + " exist at far end"); + Path base = methodPath(); + Path source = new Path(base, "source"); + Path dest = new Path(base, "dest"); + + S3AFileSystem fs = getKeepingFS(); + CreatedPaths createdPaths = createPaths(fs, source); + + // audit will find three entries + int expectedMarkerCount = createdPaths.dirs.size(); + + markerTool(fs, source, false, expectedMarkerCount); + fs.rename(source, dest); + assertIsDirectory(dest); + + // there are no markers + markerTool(fs, dest, false, 0); + LOG.info("Auditing destination paths"); + verifyRenamed(dest, createdPaths); + } + + /** + * Create a FS where only dir2 in the source tree keeps markers; + * verify all is good. + */ + @Test + public void testAuthPathIsMixed() throws Throwable { + describe("Create a source tree with mixed semantics"); + Path base = methodPath(); + Path source = new Path(base, "source"); + Path dest = new Path(base, "dest"); + Path dir2 = new Path(source, "dir2"); + S3AFileSystem mixedFSDir2 = createFS(DIRECTORY_MARKER_POLICY_AUTHORITATIVE, + dir2.toUri().toString()); + // line up for close in teardown + setMixedFS(mixedFSDir2); + // some of these paths will retain markers, some will not + CreatedPaths createdPaths = createPaths(mixedFSDir2, source); + + // markers are only under dir2 + markerTool(mixedFSDir2, toPath(source, "dir1"), false, 0); + markerTool(mixedFSDir2, source, false, expectedMarkersUnderDir2); + + // full scan of source will fail + markerTool(EXIT_NOT_ACCEPTABLE, + mixedFSDir2, source, false, 0, 0, false); + + // but add the -nonauth option and the markers under dir2 are skipped + markerTool(0, mixedFSDir2, source, false, 0, 0, true); + + // if we now rename, all will be good + LOG.info("Executing rename"); + mixedFSDir2.rename(source, dest); + assertIsDirectory(dest); + + // there are no markers + MarkerTool.ScanResult scanResult = markerTool(mixedFSDir2, dest, false, 0); + // there are exactly the files we want + Assertions.assertThat(scanResult) + .describedAs("Scan result %s", scanResult) + .extracting(s -> s.getTracker().getFilesFound()) + .isEqualTo(expectedFileCount); + verifyRenamed(dest, createdPaths); + } + + /** + * Assert that an expected number of markers were deleted. + * @param expected expected count. + * @param result scan result + */ + private static void assertMarkersDeleted(int expected, + MarkerTool.ScanResult result) { + + Assertions.assertThat(result.getPurgeSummary()) + .describedAs("Purge result of scan %s", result) + .isNotNull() + .extracting(f -> f.getMarkersDeleted()) + .isEqualTo(expected); + } + + /** + * Marker tool with no args. + */ + @Test + public void testRunNoArgs() throws Throwable { + runToFailure(EXIT_USAGE, MARKERS); + } + + @Test + public void testRunWrongBucket() throws Throwable { + runToFailure(EXIT_NOT_FOUND, MARKERS, + AUDIT, + "s3a://this-bucket-does-not-exist-hopefully"); + } + + /** + * Run with a path that doesn't exist. + */ + @Test + public void testRunUnknownPath() throws Throwable { + runToFailure(EXIT_NOT_FOUND, MARKERS, + AUDIT, + methodPath()); + } + + /** + * Having both -audit and -clean on the command line is an error. + */ + @Test + public void testRunTooManyActions() throws Throwable { + runToFailure(EXIT_USAGE, MARKERS, + AUDIT, CLEAN, + methodPath()); + } + + @Test + public void testRunAuditWithExpectedMarkers() throws Throwable { + describe("Run a verbose audit expecting some markers"); + // a run under the keeping FS will create paths + CreatedPaths createdPaths = createPaths(getKeepingFS(), methodPath()); + final File audit = tempAuditFile(); + run(MARKERS, V, + AUDIT, + m(OPT_LIMIT), 0, + m(OPT_OUT), audit, + m(OPT_EXPECTED), expectedMarkersWithBaseDir, + createdPaths.base); + expectMarkersInOutput(audit, expectedMarkersWithBaseDir); + } + + @Test + public void testRunAuditWithExcessMarkers() throws Throwable { + describe("Run a verbose audit failing as surplus markers were found"); + // a run under the keeping FS will create paths + CreatedPaths createdPaths = createPaths(getKeepingFS(), methodPath()); + final File audit = tempAuditFile(); + runToFailure(EXIT_NOT_ACCEPTABLE, MARKERS, V, + AUDIT, + m(OPT_OUT), audit, + createdPaths.base); + expectMarkersInOutput(audit, expectedMarkersWithBaseDir); + } + + @Test + public void testRunLimitedAudit() throws Throwable { + describe("Audit with a limited number of files (2)"); + CreatedPaths createdPaths = createPaths(getKeepingFS(), methodPath()); + runToFailure(EXIT_INTERRUPTED, + MARKERS, V, + m(OPT_LIMIT), 2, + CLEAN, + createdPaths.base); + run(MARKERS, V, + AUDIT, + createdPaths.base); + } + + /** + * Run an audit against the landsat bucket. + *

+ * This tests paging/scale against a larger bucket without + * worrying about setup costs. + */ + @Test + public void testRunLimitedLandsatAudit() throws Throwable { + describe("Audit a few thousand landsat objects"); + final File audit = tempAuditFile(); + + run(MARKERS, + AUDIT, + m(OPT_LIMIT), 3000, + m(OPT_OUT), audit, + LANDSAT_BUCKET); + readOutput(audit); + } + + @Test + public void testBucketInfoKeepingOnDeleting() throws Throwable { + describe("Run bucket info with the keeping config on the deleting fs"); + runS3GuardCommandToFailure(uncachedFSConfig(getDeletingFS()), + EXIT_NOT_ACCEPTABLE, + BUCKET_INFO, + m(MARKERS), DIRECTORY_MARKER_POLICY_KEEP, + methodPath()); + } + + @Test + public void testBucketInfoKeepingOnKeeping() throws Throwable { + describe("Run bucket info with the keeping config on the keeping fs"); + runS3GuardCommand(uncachedFSConfig(getKeepingFS()), + BUCKET_INFO, + m(MARKERS), DIRECTORY_MARKER_POLICY_KEEP, + methodPath()); + } + + @Test + public void testBucketInfoDeletingOnDeleting() throws Throwable { + describe("Run bucket info with the deleting config on the deleting fs"); + runS3GuardCommand(uncachedFSConfig(getDeletingFS()), + BUCKET_INFO, + m(MARKERS), DIRECTORY_MARKER_POLICY_DELETE, + methodPath()); + } + + @Test + public void testBucketInfoAuthOnAuth() throws Throwable { + describe("Run bucket info with the auth FS"); + Path base = methodPath(); + + S3AFileSystem authFS = createFS(DIRECTORY_MARKER_POLICY_AUTHORITATIVE, + base.toUri().toString()); + // line up for close in teardown + setMixedFS(authFS); + runS3GuardCommand(uncachedFSConfig(authFS), + BUCKET_INFO, + m(MARKERS), DIRECTORY_MARKER_POLICY_AUTHORITATIVE, + methodPath()); + } + + /** + * Tracker of created paths. + */ + private static final class CreatedPaths { + + private final FileSystem fs; + + private final Path base; + + private List files = new ArrayList<>(); + + private List dirs = new ArrayList<>(); + + private List emptyDirs = new ArrayList<>(); + + private List filesUnderBase = new ArrayList<>(); + + private List dirsUnderBase = new ArrayList<>(); + + private List emptyDirsUnderBase = new ArrayList<>(); + + /** + * Constructor. + * @param fs filesystem. + * @param base base directory for all creation operations. + */ + private CreatedPaths(final FileSystem fs, + final Path base) { + this.fs = fs; + this.base = base; + } + + /** + * Make a set of directories. + * @param names varargs list of paths under the base. + * @return number of entries created. + * @throws IOException failure + */ + private int dirs(String... names) throws IOException { + for (String name : names) { + mkdir(name); + } + return names.length; + } + + /** + * Create a single directory under the base. + * @param name name/relative names of the directory + * @return the path of the new entry. + */ + private Path mkdir(String name) throws IOException { + Path dir = toPath(base, name); + fs.mkdirs(dir); + dirs.add(dir); + dirsUnderBase.add(name); + return dir; + } + + /** + * Make a set of empty directories. + * @param names varargs list of paths under the base. + * @return number of entries created. + * @throws IOException failure + */ + private int emptydirs(String... names) throws IOException { + for (String name : names) { + emptydir(name); + } + return names.length; + } + + /** + * Create an empty directory. + * @param name name under the base dir + * @return the path + * @throws IOException failure + */ + private Path emptydir(String name) throws IOException { + Path dir = toPath(base, name); + fs.mkdirs(dir); + emptyDirs.add(dir); + emptyDirsUnderBase.add(name); + return dir; + } + + /** + * Make a set of files. + * @param names varargs list of paths under the base. + * @return number of entries created. + * @throws IOException failure + */ + private int files(String... names) throws IOException { + for (String name : names) { + mkfile(name); + } + return names.length; + } + + /** + * Create a 0-byte file. + * @param name name under the base dir + * @return the path + * @throws IOException failure + */ + private Path mkfile(String name) + throws IOException { + Path file = toPath(base, name); + ContractTestUtils.touch(fs, file); + files.add(file); + filesUnderBase.add(name); + return file; + } + } + + /** + * Create the "standard" test paths. + * @param fs filesystem + * @param base base dir + * @return the details on what was created. + */ + private CreatedPaths createPaths(FileSystem fs, Path base) + throws IOException { + CreatedPaths r = new CreatedPaths(fs, base); + // the directories under which we will create files, + // so expect to have markers + r.mkdir(""); + + // create the empty dirs + r.emptydir("empty"); + + // dir 1 has a file underneath + r.mkdir("dir1"); + expectedFileCount = r.files("dir1/file1"); + + expectedMarkersUnderDir1 = 1; + + + // dir2 has a subdir + r.dirs("dir2", "dir2/dir3"); + // an empty subdir + r.emptydir("dir2/empty2"); + + // and a file under itself and dir3 + expectedFileCount += r.files( + "dir2/file2", + "dir2/dir3/file3"); + + + // wrap up the expectations. + expectedMarkersUnderDir2 = 2; + expectedMarkers = expectedMarkersUnderDir1 + expectedMarkersUnderDir2; + expectedMarkersWithBaseDir = expectedMarkers + 1; + return r; + } + + /** + * Verify that all the paths renamed from the source exist + * under the destination, including all empty directories. + * @param dest destination to look under. + * @param createdPaths list of created paths. + */ + void verifyRenamed(final Path dest, + final CreatedPaths createdPaths) throws IOException { + // all leaf directories exist + for (String p : createdPaths.emptyDirsUnderBase) { + assertIsDirectory(toPath(dest, p)); + } + // non-empty dirs + for (String p : createdPaths.dirsUnderBase) { + assertIsDirectory(toPath(dest, p)); + } + // all files exist + for (String p : createdPaths.filesUnderBase) { + assertIsFile(toPath(dest, p)); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerToolRootOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerToolRootOperations.java new file mode 100644 index 0000000000000..02fec81513fca --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerToolRootOperations.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.tools; + +import java.io.File; + +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.AUDIT; +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.CLEAN; +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.MARKERS; +import static org.apache.hadoop.fs.s3a.tools.MarkerTool.OPT_OUT; + +/** + * Marker tool tests against the root FS; run in the sequential phase. + */ +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class ITestMarkerToolRootOperations extends AbstractMarkerToolTest { + + private Path rootPath; + + @Override + public void setup() throws Exception { + super.setup(); + rootPath = getFileSystem().makeQualified(new Path("/")); + } + + @Test + public void test_100_audit_root_noauth() throws Throwable { + describe("Run a verbose audit"); + final File audit = tempAuditFile(); + run(MARKERS, V, + AUDIT, + m(OPT_OUT), audit, + rootPath); + readOutput(audit); + } + + @Test + public void test_200_clean_root() throws Throwable { + describe("Clean the root path"); + final File audit = tempAuditFile(); + run(MARKERS, V, + CLEAN, + m(OPT_OUT), audit, + rootPath); + readOutput(audit); + } + +} From 092bfe7c8e9736efb0dbe3d39af77c4bfffa6ef2 Mon Sep 17 00:00:00 2001 From: bshashikant Date: Mon, 17 Aug 2020 12:56:13 +0530 Subject: [PATCH 078/335] HDFS-15483. Ordered snapshot deletion: Disallow rename between two snapshottable directories. (#2172) * HDFS-15483. Ordered snapshot deletion: Disallow rename between two snapshottable directories. * Addressed review comments. * Rebased to latest trunk and added snapshotTrashRoot config check. * Addressed review comments. * Removede empty line added in SnapshotManager.Java. * Addressed whitespace issues. --- .../hdfs/server/namenode/FSDirRenameOp.java | 27 ++++- .../hdfs/server/namenode/FSNamesystem.java | 16 ++- .../namenode/snapshot/SnapshotManager.java | 34 ++++-- ...TestRenameWithOrderedSnapshotDeletion.java | 110 ++++++++++++++++++ 4 files changed, 172 insertions(+), 15 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithOrderedSnapshotDeletion.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java index a0c1e3a8814e3..7396519e90af2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.Time; @@ -193,7 +194,7 @@ static INodesInPath unprotectedRenameTo(FSDirectory fsd, } validateNestSnapshot(fsd, src, dstParent.asDirectory(), snapshottableDirs); - + checkUnderSameSnapshottableRoot(fsd, srcIIP, dstIIP); fsd.ezManager.checkMoveValidity(srcIIP, dstIIP); // Ensure dst has quota to accommodate rename verifyFsLimitsForRename(fsd, srcIIP, dstIIP); @@ -407,6 +408,7 @@ static RenameResult unprotectedRenameTo(FSDirectory fsd, validateNestSnapshot(fsd, src, dstParent.asDirectory(), srcSnapshottableDirs); + checkUnderSameSnapshottableRoot(fsd, srcIIP, dstIIP); // Ensure dst has quota to accommodate rename verifyFsLimitsForRename(fsd, srcIIP, dstIIP); @@ -821,6 +823,29 @@ void updateQuotasInSourceTree(BlockStoragePolicySuite bsps) { } } + private static void checkUnderSameSnapshottableRoot( + FSDirectory fsd, INodesInPath srcIIP, INodesInPath dstIIP) + throws IOException { + // Ensure rename out of a snapshottable root is not permitted if ordered + // snapshot deletion feature is enabled + SnapshotManager snapshotManager = fsd.getFSNamesystem(). + getSnapshotManager(); + if (snapshotManager.isSnapshotDeletionOrdered() && fsd.getFSNamesystem() + .isSnapshotTrashRootEnabled()) { + INodeDirectory srcRoot = snapshotManager. + getSnapshottableAncestorDir(srcIIP); + INodeDirectory dstRoot = snapshotManager. + getSnapshottableAncestorDir(dstIIP); + // Ensure snapshoottable root for both src and dest are same. + if (srcRoot != dstRoot) { + String errMsg = "Source " + srcIIP.getPath() + + " and dest " + dstIIP.getPath() + " are not under " + + "the same snapshot root."; + throw new SnapshotException(errMsg); + } + } + } + private static RenameResult createRenameResult(FSDirectory fsd, INodesInPath dst, boolean filesDeleted, BlocksMapUpdateInfo collectedBlocks) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 0a273f0a6d1d3..15bf6b16b569d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -384,9 +384,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, .getLogger(FSNamesystem.class.getName()); // The following are private configurations - static final String DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED = + public static final String DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED = "dfs.namenode.snapshot.trashroot.enabled"; - static final boolean DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT = false; + public static final boolean DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT + = false; private final MetricsRegistry registry = new MetricsRegistry("FSNamesystem"); @Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics = @@ -468,6 +469,7 @@ private void logAuditEvent(boolean succeeded, private final UserGroupInformation fsOwner; private final String supergroup; private final boolean standbyShouldCheckpoint; + private final boolean isSnapshotTrashRootEnabled; private final int snapshotDiffReportLimit; private final int blockDeletionIncrement; @@ -882,6 +884,9 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { // Get the checksum type from config String checksumTypeStr = conf.get(DFS_CHECKSUM_TYPE_KEY, DFS_CHECKSUM_TYPE_DEFAULT); + this.isSnapshotTrashRootEnabled = conf.getBoolean( + DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED, + DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT); DataChecksum.Type checksumType; try { checksumType = DataChecksum.Type.valueOf(checksumTypeStr); @@ -909,8 +914,7 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, ""), blockManager.getStoragePolicySuite().getDefaultPolicy().getId(), - conf.getBoolean(DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED, - DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT)); + isSnapshotTrashRootEnabled); this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, DFS_NAMENODE_MAX_OBJECTS_DEFAULT); @@ -1054,6 +1058,10 @@ public int getMaxListOpenFilesResponses() { return maxListOpenFilesResponses; } + boolean isSnapshotTrashRootEnabled() { + return isSnapshotTrashRootEnabled; + } + void lockRetryCache() { if (retryCache != null) { retryCache.lock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 3866125503325..b5b0971298976 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -363,21 +363,35 @@ void assertFirstSnapshot(INodeDirectory dir, * @param iip INodesInPath for the directory to get snapshot root. * @return the snapshot root INodeDirectory */ + public INodeDirectory checkAndGetSnapshottableAncestorDir( + final INodesInPath iip) throws IOException { + final INodeDirectory dir = getSnapshottableAncestorDir(iip); + if (dir == null) { + throw new SnapshotException("Directory is neither snapshottable nor" + + " under a snap root!"); + } + return dir; + } + public INodeDirectory getSnapshottableAncestorDir(final INodesInPath iip) throws IOException { final String path = iip.getPath(); - final INodeDirectory dir = INodeDirectory.valueOf(iip.getLastINode(), path); + final INode inode = iip.getLastINode(); + final INodeDirectory dir; + if (inode instanceof INodeDirectory) { + dir = INodeDirectory.valueOf(inode, path); + } else { + dir = INodeDirectory.valueOf(iip.getINode(-2), iip.getParentPath()); + } if (dir.isSnapshottable()) { return dir; - } else { - for (INodeDirectory snapRoot : this.snapshottables.values()) { - if (dir.isAncestorDirectory(snapRoot)) { - return snapRoot; - } + } + for (INodeDirectory snapRoot : this.snapshottables.values()) { + if (dir.isAncestorDirectory(snapRoot)) { + return snapRoot; } - throw new SnapshotException("Directory is neither snapshottable nor" + - " under a snap root!"); } + return null; } public boolean isDescendantOfSnapshotRoot(INodeDirectory dir) { @@ -641,7 +655,7 @@ public SnapshotDiffReport diff(final INodesInPath iip, // All the check for path has been included in the valueOf method. INodeDirectory snapshotRootDir; if (this.snapshotDiffAllowSnapRootDescendant) { - snapshotRootDir = getSnapshottableAncestorDir(iip); + snapshotRootDir = checkAndGetSnapshottableAncestorDir(iip); } else { snapshotRootDir = getSnapshottableRoot(iip); } @@ -674,7 +688,7 @@ public SnapshotDiffReportListing diff(final INodesInPath iip, // All the check for path has been included in the valueOf method. INodeDirectory snapshotRootDir; if (this.snapshotDiffAllowSnapRootDescendant) { - snapshotRootDir = getSnapshottableAncestorDir(iip); + snapshotRootDir = checkAndGetSnapshottableAncestorDir(iip); } else { snapshotRootDir = getSnapshottableRoot(iip); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithOrderedSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithOrderedSnapshotDeletion.java new file mode 100644 index 0000000000000..052610baec873 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithOrderedSnapshotDeletion.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.snapshot; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + + +import java.io.IOException; + +import static org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager.DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED; +import static org.apache.hadoop.hdfs.server.namenode.FSNamesystem.DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED; +/** + * Test Rename with ordered snapshot deletion. + */ +public class TestRenameWithOrderedSnapshotDeletion { + private final Path snapshottableDir + = new Path("/" + getClass().getSimpleName()); + private DistributedFileSystem hdfs; + private MiniDFSCluster cluster; + + @Before + public void setUp() throws Exception { + final Configuration conf = new Configuration(); + conf.setBoolean(DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED, true); + conf.setBoolean(DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED, true); + + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster.waitActive(); + hdfs = cluster.getFileSystem(); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Test(timeout = 60000) + public void testRename() throws Exception { + final Path dir1 = new Path("/dir1"); + final Path dir2 = new Path("/dir2"); + final Path sub0 = new Path(snapshottableDir, "sub0"); + final Path sub1 = new Path(snapshottableDir, "sub1"); + hdfs.mkdirs(sub0); + hdfs.mkdirs(dir2); + final Path file1 = new Path(dir1, "file1"); + final Path file2 = new Path(sub0, "file2"); + hdfs.mkdirs(snapshottableDir); + hdfs.mkdirs(dir1); + hdfs.mkdirs(dir2); + hdfs.mkdirs(sub0); + DFSTestUtil.createFile(hdfs, file1, 0, (short) 1, 0); + DFSTestUtil.createFile(hdfs, file2, 0, (short) 1, 0); + hdfs.allowSnapshot(snapshottableDir); + // rename from non snapshottable dir to snapshottable dir should fail + validateRename(file1, sub0); + hdfs.createSnapshot(snapshottableDir, "s0"); + validateRename(file1, sub0); + // rename across non snapshottable dirs should work + hdfs.rename(file1, dir2); + // rename beyond snapshottable root should fail + validateRename(file2, dir1); + // rename within snapshottable root should work + hdfs.rename(file2, snapshottableDir); + + // rename dirs outside snapshottable root should work + hdfs.rename(dir2, dir1); + // rename dir into snapshottable root should fail + validateRename(dir1, snapshottableDir); + // rename dir outside snapshottable root should fail + validateRename(sub0, dir2); + // rename dir within snapshottable root should work + hdfs.rename(sub0, sub1); + } + + private void validateRename(Path src, Path dest) { + try { + hdfs.rename(src, dest); + Assert.fail("Expected exception not thrown."); + } catch (IOException ioe) { + Assert.assertTrue(ioe.getMessage().contains("are not under the" + + " same snapshot root.")); + } + } +} From 4a7deae478947c2ec250854ce6ecdf710d7637c3 Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Mon, 17 Aug 2020 16:41:15 +0000 Subject: [PATCH 079/335] YARN-10391. --module-gpu functionality is broken in container-executor. Contributed by Eric Badger --- .../src/main/native/container-executor/impl/main.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 9555f80e177d1..ff59b96d23362 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -658,15 +658,15 @@ int main(int argc, char **argv) { assert_valid_setup(argv[0]); int operation = -1; - int ret = validate_arguments(argc, argv, &operation); + int exit_code = 0; + exit_code = validate_arguments(argc, argv, &operation); - if (ret != 0) { - flush_and_close_log_files(); - return ret; + if (exit_code != 0 || operation == -1) { + // if operation is still -1, the work was done in validate_arguments + // e.g. for --module-gpu + goto cleanup; } - int exit_code = 0; - switch (operation) { case CHECK_SETUP: //we already did this @@ -831,6 +831,7 @@ int main(int argc, char **argv) { break; } +cleanup: if (exit_code) { fprintf(ERRORFILE, "Nonzero exit code=%d, error message='%s'\n", exit_code, get_error_message(exit_code)); From b367942fe49364b2a1643d23f9699f95654735dc Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Mon, 17 Aug 2020 22:47:18 +0530 Subject: [PATCH 080/335] Upgrade store REST API version to 2019-12-12 - Contributed by Sneha Vijayarajan --- .../java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index e1ea75e4756de..3ee2d7e8e19d3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -62,7 +62,7 @@ public class AbfsClient implements Closeable { public static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; - private final String xMsVersion = "2018-11-09"; + private final String xMsVersion = "2019-12-12"; private final ExponentialRetryPolicy retryPolicy; private final String filesystem; private final AbfsConfiguration abfsConfiguration; From fefacf2578e58cf7bf96d920f5b01dba66909ca2 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 17 Aug 2020 20:08:58 -0700 Subject: [PATCH 081/335] HADOOP-17205. Move personality file from Yetus to Hadoop repository (#2226) Signed-off-by: Akira Ajisaka --- Jenkinsfile | 2 +- dev-support/bin/hadoop.sh | 563 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 564 insertions(+), 1 deletion(-) create mode 100755 dev-support/bin/hadoop.sh diff --git a/Jenkinsfile b/Jenkinsfile index 302dbd04d6c9e..4d2b4529945c9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -96,8 +96,8 @@ pipeline { YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") # our project defaults come from a personality file - # which will get loaded automatically by setting the project name YETUS_ARGS+=("--project=hadoop") + YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh") # lots of different output formats YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh new file mode 100755 index 0000000000000..88b6005b7beb7 --- /dev/null +++ b/dev-support/bin/hadoop.sh @@ -0,0 +1,563 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# SHELLDOC-IGNORE +# +# Override these to match Apache Hadoop's requirements +personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + BUILDTOOL=maven + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=trunk + #shellcheck disable=SC2034 + PATCH_NAMING_RULE="https://wiki.apache.org/hadoop/HowToContribute" + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO_DEFAULT="apache/hadoop" + + HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)} + if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then + HADOOP_HOMEBREW_DIR=/usr/local + fi +} + +function personality_parse_args +{ + declare i + + for i in "$@"; do + case ${i} in + --hadoop-isal-prefix=*) + delete_parameter "${i}" + ISAL_HOME=${i#*=} + ;; + --hadoop-openssl-prefix=*) + delete_parameter "${i}" + OPENSSL_HOME=${i#*=} + ;; + --hadoop-snappy-prefix=*) + delete_parameter "${i}" + SNAPPY_HOME=${i#*=} + ;; + esac + done +} + +## @description Calculate the actual module ordering +## @audience private +## @stability evolving +## @param ordering +function hadoop_order +{ + declare ordering=$1 + declare hadoopm + + if [[ ${ordering} = normal ]]; then + hadoopm="${CHANGED_MODULES[*]}" + elif [[ ${ordering} = union ]]; then + hadoopm="${CHANGED_UNION_MODULES}" + elif [[ ${ordering} = mvnsrc ]]; then + hadoopm="${MAVEN_SRC_MODULES[*]}" + elif [[ ${ordering} = mvnsrctest ]]; then + hadoopm="${MAVEN_SRCTEST_MODULES[*]}" + else + hadoopm="${ordering}" + fi + echo "${hadoopm}" +} + +## @description Determine if it is safe to run parallel tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_test_parallel +{ + if [[ -f "${BASEDIR}/pom.xml" ]]; then + HADOOP_VERSION=$(grep '' "${BASEDIR}/pom.xml" \ + | head -1 \ + | "${SED}" -e 's|^ *||' -e 's|.*$||' \ + | cut -f1 -d- ) + export HADOOP_VERSION + else + return 1 + fi + + hmajor=${HADOOP_VERSION%%\.*} + hmajorminor=${HADOOP_VERSION%\.*} + hminor=${hmajorminor##*\.} + # ... and just for reference + #hmicro=${HADOOP_VERSION##*\.} + + # Apache Hadoop v2.8.0 was the first one to really + # get working parallel unit tests + if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then + return 1 + fi + + return 0 +} + +## @description Install extra modules for unit tests +## @audience private +## @stability evolving +## @param ordering +function hadoop_unittest_prereqs +{ + declare input=$1 + declare mods + declare need_common=0 + declare building_common=0 + declare module + declare flags + declare fn + + # prior to running unit tests, hdfs needs libhadoop.so built + # if we're building root, then this extra work is moot + + #shellcheck disable=SC2086 + mods=$(hadoop_order ${input}) + + for module in ${mods}; do + if [[ ${module} = hadoop-hdfs-project* ]]; then + need_common=1 + elif [[ ${module} = hadoop-common-project/hadoop-common + || ${module} = hadoop-common-project ]]; then + building_common=1 + elif [[ ${module} = . ]]; then + return + fi + done + + # Windows builds *ALWAYS* need hadoop-common compiled + case ${OSTYPE} in + Windows_NT|CYGWIN*|MINGW*|MSYS*) + need_common=1 + ;; + esac + + if [[ ${need_common} -eq 1 + && ${building_common} -eq 0 ]]; then + echo "unit test pre-reqs:" + module="hadoop-common-project/hadoop-common" + fn=$(module_file_fragment "${module}") + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + pushd "${BASEDIR}/${module}" >/dev/null || return 1 + # shellcheck disable=SC2086 + echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} + popd >/dev/null || return 1 + fi +} + +## @description Calculate the flags/settings for yarn-ui v2 build +## @description based upon the OS +## @audience private +## @stability evolving +function yarn_ui2_flag +{ + + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + # Now it only tested on Linux/OSX, don't enable the profile on + # windows until it get verified + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo -Pyarn-ui + ;; + Darwin) + echo -Pyarn-ui + ;; + *) + # Do nothing + ;; + esac +} + +## @description Calculate the flags/settings for native code +## @description based upon the OS +## @audience private +## @stability evolving +function hadoop_native_flags +{ + if [[ ${BUILD_NATIVE} != true ]]; then + return + fi + + declare -a args + + # Based upon HADOOP-11937 + # + # Some notes: + # + # - getting fuse to compile on anything but Linux + # is always tricky. + # - Darwin assumes homebrew is in use. + # - HADOOP-12027 required for bzip2 on OS X. + # - bzip2 is broken in lots of places + # (the shared library is considered experimental) + # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 + # + + args=("-Drequire.test.libhadoop") + + if [[ -d "${ISAL_HOME}/include" ]]; then + args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}") + fi + + if [[ -d "${OPENSSL_HOME}/include" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then + args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/") + fi + + if [[ -d "${SNAPPY_HOME}/include" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}") + elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then + args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy") + fi + + case ${OSTYPE} in + Linux) + # shellcheck disable=SC2086 + echo \ + -Pnative \ + -Drequire.fuse \ + -Drequire.openssl \ + -Drequire.snappy \ + -Drequire.valgrind \ + -Drequire.zstd \ + "${args[@]}" + ;; + Darwin) + echo \ + "${args[@]}" \ + -Pnative \ + -Drequire.snappy \ + -Drequire.openssl + ;; + Windows_NT|CYGWIN*|MINGW*|MSYS*) + echo \ + "${args[@]}" \ + -Drequire.snappy -Drequire.openssl -Pnative-win + ;; + *) + echo \ + "${args[@]}" + ;; + esac +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + declare repostatus=$1 + declare testtype=$2 + declare extra="" + declare ordering="normal" + declare needflags=false + declare foundbats=false + declare flags + declare fn + declare i + declare hadoopm + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + case ${testtype} in + asflicense) + # this is very fast and provides the full path if we do it from + # the root of the source + personality_enqueue_module . + return + ;; + checkstyle) + ordering="union" + extra="-DskipTests" + ;; + compile) + ordering="union" + extra="-DskipTests" + needflags=true + + # if something in common changed, we build the whole world + if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then + yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " + ordering="." + fi + ;; + distclean) + ordering="." + extra="-DskipTests" + ;; + javadoc) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then + echo "javadoc pre-reqs:" + for i in hadoop-project \ + hadoop-common-project/hadoop-annotations; do + fn=$(module_file_fragment "${i}") + pushd "${BASEDIR}/${i}" >/dev/null || return 1 + echo "cd ${i}" + echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install + popd >/dev/null || return 1 + done + fi + extra="-Pdocs -DskipTests" + ;; + mvneclipse) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + mvninstall) + extra="-DskipTests" + if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then + ordering=. + fi + ;; + mvnsite) + if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + ;; + unit) + if [[ "${BUILDMODE}" = full ]]; then + ordering=mvnsrc + elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then + ordering=. + fi + + if [[ ${TEST_PARALLEL} = "true" ]] ; then + if hadoop_test_parallel; then + extra="-Pparallel-tests" + if [[ -n ${TEST_THREADS:-} ]]; then + extra="${extra} -DtestsThreadCount=${TEST_THREADS}" + fi + fi + fi + needflags=true + hadoop_unittest_prereqs "${ordering}" + + if ! verify_needed_test javac; then + yetus_debug "hadoop: javac not requested" + if ! verify_needed_test native; then + yetus_debug "hadoop: native not requested" + yetus_debug "hadoop: adding -DskipTests to unit test" + extra="-DskipTests" + fi + fi + + for i in "${CHANGED_FILES[@]}"; do + if [[ "${i}" =~ \.bats ]]; then + foundbats=true + fi + done + + if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then + yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile" + extra="${extra} -P!shelltest" + else + extra="${extra} -Pshelltest" + fi + ;; + *) + extra="-DskipTests" + ;; + esac + + if [[ ${needflags} = true ]]; then + flags="$(hadoop_native_flags) $(yarn_ui2_flag)" + extra="${extra} ${flags}" + fi + + extra="-Ptest-patch ${extra}" + for module in $(hadoop_order ${ordering}); do + # shellcheck disable=SC2086 + personality_enqueue_module ${module} ${extra} + done +} + +## @description Add tests based upon personality needs +## @audience private +## @stability evolving +## @param filename +function personality_file_tests +{ + declare filename=$1 + + yetus_debug "Using Hadoop-specific personality_file_tests" + + if [[ ${filename} =~ src/main/webapp ]]; then + yetus_debug "tests/webapp: ${filename}" + add_test shadedclient + elif [[ ${filename} =~ \.sh + || ${filename} =~ \.cmd + || ${filename} =~ src/scripts + || ${filename} =~ src/test/scripts + || ${filename} =~ src/main/bin + || ${filename} =~ shellprofile\.d + || ${filename} =~ src/main/conf + ]]; then + yetus_debug "tests/shell: ${filename}" + add_test mvnsite + add_test unit + elif [[ ${filename} =~ \.md$ + || ${filename} =~ \.md\.vm$ + || ${filename} =~ src/site + ]]; then + yetus_debug "tests/site: ${filename}" + add_test mvnsite + elif [[ ${filename} =~ \.c$ + || ${filename} =~ \.cc$ + || ${filename} =~ \.h$ + || ${filename} =~ \.hh$ + || ${filename} =~ \.proto$ + || ${filename} =~ \.cmake$ + || ${filename} =~ CMakeLists.txt + ]]; then + yetus_debug "tests/units: ${filename}" + add_test compile + add_test cc + add_test mvnsite + add_test javac + add_test unit + elif [[ ${filename} =~ build.xml$ + || ${filename} =~ pom.xml$ + || ${filename} =~ \.java$ + || ${filename} =~ src/main + ]]; then + yetus_debug "tests/javadoc+units: ${filename}" + add_test compile + add_test javac + add_test javadoc + add_test mvninstall + add_test mvnsite + add_test unit + add_test shadedclient + fi + + # if we change anything in here, e.g. the test scripts + # then run the client artifact tests + if [[ ${filename} =~ hadoop-client-modules ]]; then + add_test shadedclient + fi + + if [[ ${filename} =~ src/test ]]; then + yetus_debug "tests: src/test" + add_test unit + fi + + if [[ ${filename} =~ \.java$ ]]; then + add_test findbugs + fi +} + +## @description Image to print on success +## @audience private +## @stability evolving +function hadoop_console_success +{ + printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; + printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; + printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; + printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; + printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; + printf "ICB8bXwgICB8bXwgIAo" +} + +################################################### +# Hadoop project specific check of IT for shaded artifacts + +add_test_type shadedclient + +## @description check for test modules and add test/plugins as needed +## @audience private +## @stability evolving +function shadedclient_initialize +{ + maven_add_install shadedclient +} + +## @description build client facing shaded artifacts and test them +## @audience private +## @stability evolving +## @param repostatus +function shadedclient_rebuild +{ + declare repostatus=$1 + declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt" + declare module + declare -a modules=() + + if [[ ${OSTYPE} = Windows_NT || + ${OSTYPE} =~ ^CYGWIN.* || + ${OSTYPE} =~ ^MINGW32.* || + ${OSTYPE} =~ ^MSYS.* ]]; then + echo "hadoop personality: building on windows, skipping check of client artifacts." + return 0 + fi + + yetus_debug "hadoop personality: seeing if we need the test of client artifacts." + for module in hadoop-client-modules/hadoop-client-check-invariants \ + hadoop-client-modules/hadoop-client-check-test-invariants \ + hadoop-client-modules/hadoop-client-integration-tests; do + if [ -d "${module}" ]; then + yetus_debug "hadoop personality: test module '${module}' is present." + modules+=(-pl "${module}") + fi + done + if [ ${#modules[@]} -eq 0 ]; then + echo "hadoop personality: no test modules present, skipping check of client artifacts." + return 0 + fi + + big_console_header "Checking client artifacts on ${repostatus}" + + echo_and_redirect "${logfile}" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ + "${modules[@]}" \ + -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true + + count=$("${GREP}" -c '\[ERROR\]' "${logfile}") + if [[ ${count} -gt 0 ]]; then + add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts." + return 1 + fi + + add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts." + return 0 +} From b65e43fe386c9e9cf056f1c460b6c4a2605aeebe Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 18 Aug 2020 03:28:19 -0700 Subject: [PATCH 082/335] HDFS-15525. Make trash root inside each snapshottable directory for WebHDFS (#2220) --- .../web/resources/NamenodeWebHdfsMethods.java | 46 +++++++++++++++++-- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 28 +++++++++++ 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 2423a037c8fd0..9baed4f06730c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -1345,19 +1345,55 @@ protected Response get( } } + /** + * Get the snapshot root of a given file or directory if it exists. + * e.g. if /snapdir1 is a snapshottable directory and path given is + * /snapdir1/path/to/file, this method would return /snapdir1 + * @param pathStr String of path to a file or a directory. + * @return Not null if found in a snapshot root directory. + * @throws IOException + */ + String getSnapshotRoot(String pathStr) throws IOException { + SnapshottableDirectoryStatus[] dirStatusList = + getRpcClientProtocol().getSnapshottableDirListing(); + if (dirStatusList == null) { + return null; + } + for (SnapshottableDirectoryStatus dirStatus : dirStatusList) { + String currDir = dirStatus.getFullPath().toString(); + if (pathStr.startsWith(currDir)) { + return currDir; + } + } + return null; + } + private String getTrashRoot(Configuration conf, String fullPath) throws IOException { - UserGroupInformation ugi= UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String parentSrc = getParent(fullPath); + String ssTrashRoot = ""; + boolean isSnapshotTrashRootEnabled = getRpcClientProtocol() + .getServerDefaults().getSnapshotTrashRootEnabled(); + if (isSnapshotTrashRootEnabled) { + String ssRoot = getSnapshotRoot(fullPath); + if (ssRoot != null) { + ssTrashRoot = DFSUtilClient.getSnapshotTrashRoot(ssRoot, ugi); + } + } EncryptionZone ez = getRpcClientProtocol().getEZForPath( parentSrc != null ? parentSrc : fullPath); - String trashRoot; + String ezTrashRoot = ""; if (ez != null) { - trashRoot = DFSUtilClient.getEZTrashRoot(ez, ugi); + ezTrashRoot = DFSUtilClient.getEZTrashRoot(ez, ugi); + } + // Choose the longest path + if (ssTrashRoot.isEmpty() && ezTrashRoot.isEmpty()) { + return DFSUtilClient.getTrashRoot(conf, ugi); } else { - trashRoot = DFSUtilClient.getTrashRoot(conf, ugi); + return ssTrashRoot.length() > ezTrashRoot.length() ? + ssTrashRoot : ezTrashRoot; } - return trashRoot; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 69a0e600ffb62..3c8a92eee12d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -1569,6 +1569,34 @@ public void testGetTrashRoot() throws Exception { assertEquals(expectedPath.toUri().getPath(), trashPath.toUri().getPath()); } + @Test + public void testGetSnapshotTrashRoot() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + final String currentUser = + UserGroupInformation.getCurrentUser().getShortUserName(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsConstants.WEBHDFS_SCHEME); + Path ssDir1 = new Path("/ssDir1"); + assertTrue(webFS.mkdirs(ssDir1)); + + Path trashPath = webFS.getTrashRoot(ssDir1); + Path expectedPath = new Path(FileSystem.USER_HOME_PREFIX, + new Path(currentUser, FileSystem.TRASH_PREFIX)); + assertEquals(expectedPath.toUri().getPath(), trashPath.toUri().getPath()); + // Enable snapshot + webFS.allowSnapshot(ssDir1); + Path trashPathAfter = webFS.getTrashRoot(ssDir1); + Path expectedPathAfter = new Path(ssDir1, + new Path(FileSystem.TRASH_PREFIX, currentUser)); + assertEquals(expectedPathAfter.toUri().getPath(), + trashPathAfter.toUri().getPath()); + // Cleanup + webFS.disallowSnapshot(ssDir1); + webFS.delete(ssDir1, true); + } + @Test public void testGetEZTrashRoot() throws Exception { final Configuration conf = WebHdfsTestUtil.createConf(); From 82ec28f4421c162a505ba5e5b329e4be199878a7 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 19 Aug 2020 11:54:48 +0530 Subject: [PATCH 083/335] YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke. --- .../scheduler/capacity/ParentQueue.java | 14 +- .../scheduler/capacity/TestParentQueue.java | 134 ++++++++++++++++-- 2 files changed, 132 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index bbb80ba73361c..923e687500c14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -1123,8 +1123,18 @@ private void deriveCapacityFromAbsoluteConfigurations(String label, if (childQueue instanceof LeafQueue) { LeafQueue leafQueue = (LeafQueue) childQueue; CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - int maxApplications = (int) (conf.getMaximumSystemApplications() - * childQueue.getQueueCapacities().getAbsoluteCapacity(label)); + int maxApplications = + conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath()); + if (maxApplications < 0) { + int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); + if (maxGlobalPerQueueApps > 0) { + maxApplications = (int) (maxGlobalPerQueueApps * + childQueue.getQueueCapacities().getAbsoluteCapacity(label)); + } else { + maxApplications = (int) (conf.getMaximumSystemApplications() + * childQueue.getQueueCapacities().getAbsoluteCapacity(label)); + } + } leafQueue.setMaxApplications(maxApplications); int maxApplicationsPerUser = Math.min(maxApplications, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 0560d595a6323..9ed0388aec1a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -108,15 +108,17 @@ public void setUp() throws Exception { private static final String A = "a"; private static final String B = "b"; + private static final String Q_A = + CapacitySchedulerConfiguration.ROOT + "." + A; + private static final String Q_B = + CapacitySchedulerConfiguration.ROOT + "." + B; private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 30); - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; conf.setCapacity(Q_B, 70); LOG.info("Setup top-level queues a and b"); @@ -128,11 +130,9 @@ private void setupSingleLevelQueuesWithAbsoluteResource( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B}); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setMinimumResourceRequirement("", Q_A, QUEUE_A_RESOURCE); - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; conf.setMinimumResourceRequirement("", Q_B, QUEUE_B_RESOURCE); @@ -368,9 +368,7 @@ public void testSingleLevelQueues() throws Exception { public void testSingleLevelQueuesPrecision() throws Exception { // Setup queue configs setupSingleLevelQueues(csConf); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + "a"; csConf.setCapacity(Q_A, 30); - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + "b"; csConf.setCapacity(Q_B, 70.5F); CSQueueStore queues = new CSQueueStore(); @@ -434,10 +432,8 @@ private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) { // Define top-level queues csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 10); - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; conf.setCapacity(Q_B, 50); final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; @@ -656,7 +652,6 @@ public void testQueueCapacitySettingChildZero() throws Exception { setupMultiLevelQueues(csConf); // set child queues capacity to 0 when parents not 0 - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; csConf.setCapacity(Q_B + "." + B1, 0); csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); @@ -673,9 +668,7 @@ public void testQueueCapacitySettingParentZero() throws Exception { setupMultiLevelQueues(csConf); // set parent capacity to 0 when child not 0 - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; csConf.setCapacity(Q_B, 0); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); CSQueueStore queues = new CSQueueStore(); @@ -690,13 +683,11 @@ public void testQueueCapacityZero() throws Exception { setupMultiLevelQueues(csConf); // set parent and child capacity to 0 - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; csConf.setCapacity(Q_B, 0); csConf.setCapacity(Q_B + "." + B1, 0); csConf.setCapacity(Q_B + "." + B2, 0); csConf.setCapacity(Q_B + "." + B3, 0); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; csConf.setCapacity(Q_A, 60); CSQueueStore queues = new CSQueueStore(); @@ -1029,10 +1020,125 @@ public void testAbsoluteResourceWithChangeInClusterResource() QUEUE_B_RESOURCE_70PERC); } + @Test + public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception { + // Setup queue configs + setupSingleLevelQueuesWithAbsoluteResource(csConf); + + CSQueueStore queues = new CSQueueStore(); + CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, + null, CapacitySchedulerConfiguration.ROOT, queues, queues, + TestUtils.spyHook); + + // Setup some nodes + int numNodes = 2; + final long memoryPerNode = (QUEUE_A_RESOURCE.getMemorySize() + + QUEUE_B_RESOURCE.getMemorySize()) / numNodes; + int coresPerNode = (QUEUE_A_RESOURCE.getVirtualCores() + + QUEUE_B_RESOURCE.getVirtualCores()) / numNodes; + + Resource clusterResource = Resources.createResource( + numNodes * memoryPerNode, numNodes * coresPerNode); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + // Start testing + // Only MaximumSystemApplications is set in csConf + LeafQueue a = (LeafQueue) queues.get(A); + LeafQueue b = (LeafQueue) queues.get(B); + + float queueAScale = (float) QUEUE_A_RESOURCE.getMemorySize() / + (float) clusterResource.getMemorySize(); + float queueBScale = (float) QUEUE_B_RESOURCE.getMemorySize() / + (float) clusterResource.getMemorySize(); + + assertEquals(queueAScale, a.getQueueCapacities().getCapacity(), + DELTA); + assertEquals(1f, a.getQueueCapacities().getMaximumCapacity(), + DELTA); + assertEquals(queueAScale, a.getQueueCapacities().getAbsoluteCapacity(), + DELTA); + assertEquals(1f, + a.getQueueCapacities().getAbsoluteMaximumCapacity(), DELTA); + assertEquals((int) (csConf.getMaximumSystemApplications() * queueAScale), + a.getMaxApplications()); + assertEquals(a.getMaxApplications(), a.getMaxApplicationsPerUser()); + + assertEquals(queueBScale, + b.getQueueCapacities().getCapacity(), DELTA); + assertEquals(1f, + b.getQueueCapacities().getMaximumCapacity(), DELTA); + assertEquals(queueBScale, + b.getQueueCapacities().getAbsoluteCapacity(), DELTA); + assertEquals(1f, + b.getQueueCapacities().getAbsoluteMaximumCapacity(), DELTA); + assertEquals((int) (csConf.getMaximumSystemApplications() * queueBScale), + b.getMaxApplications()); + assertEquals(b.getMaxApplications(), b.getMaxApplicationsPerUser()); + + // Set GlobalMaximumApplicationsPerQueue in csConf + csConf.setGlobalMaximumApplicationsPerQueue(20000); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + assertEquals((int) (csConf.getGlobalMaximumApplicationsPerQueue() * + queueAScale), a.getMaxApplications()); + assertEquals(a.getMaxApplications(), a.getMaxApplicationsPerUser()); + assertEquals((int) (csConf.getGlobalMaximumApplicationsPerQueue() * + queueBScale), b.getMaxApplications()); + assertEquals(b.getMaxApplications(), b.getMaxApplicationsPerUser()); + + // Set MaximumApplicationsPerQueue in csConf + int queueAMaxApplications = 30000; + int queueBMaxApplications = 30000; + csConf.set("yarn.scheduler.capacity." + Q_A + ".maximum-applications", + Integer.toString(queueAMaxApplications)); + csConf.set("yarn.scheduler.capacity." + Q_B + ".maximum-applications", + Integer.toString(queueBMaxApplications)); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + assertEquals(queueAMaxApplications, a.getMaxApplications()); + assertEquals(a.getMaxApplications(), a.getMaxApplicationsPerUser()); + assertEquals(queueBMaxApplications, b.getMaxApplications()); + assertEquals(b.getMaxApplications(), b.getMaxApplicationsPerUser()); + + // Extra cases for testing maxApplicationsPerUser + int halfPercent = 50; + int oneAndQuarterPercent = 125; + a.getUsersManager().setUserLimit(halfPercent); + b.getUsersManager().setUserLimit(oneAndQuarterPercent); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + assertEquals(a.getMaxApplications() * halfPercent / 100, + a.getMaxApplicationsPerUser()); + // Q_B's limit per user shouldn't be greater + // than the whole queue's application limit + assertEquals(b.getMaxApplications(), b.getMaxApplicationsPerUser()); + + float userLimitFactorQueueA = 0.9f; + float userLimitFactorQueueB = 1.1f; + a.getUsersManager().setUserLimit(halfPercent); + a.getUsersManager().setUserLimitFactor(userLimitFactorQueueA); + b.getUsersManager().setUserLimit(100); + b.getUsersManager().setUserLimitFactor(userLimitFactorQueueB); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + assertEquals((int) (a.getMaxApplications() * halfPercent * + userLimitFactorQueueA / 100), a.getMaxApplicationsPerUser()); + // Q_B's limit per user shouldn't be greater + // than the whole queue's application limit + assertEquals(b.getMaxApplications(), b.getMaxApplicationsPerUser()); + + } + @After public void tearDown() throws Exception { } - + private ResourceLimits anyResourceLimits() { return any(ResourceLimits.class); } From dd013f2fdf1ecbeb6c877e26951cd0d8922058b0 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Wed, 19 Aug 2020 09:30:41 -0700 Subject: [PATCH 084/335] HDFS-15533: Provide DFS API compatible class, but use ViewFileSystemOverloadScheme inside. (#2229). Contributed by Uma Maheswara Rao G. --- .../org/apache/hadoop/fs/FsConstants.java | 1 - .../apache/hadoop/fs/viewfs/InodeTree.java | 7 +- .../hadoop/fs/viewfs/ViewFileSystem.java | 11 +- .../viewfs/ViewFileSystemOverloadScheme.java | 75 +- .../hadoop/hdfs/DistributedFileSystem.java | 14 +- .../hdfs/ViewDistributedFileSystem.java | 2307 +++++++++++++++++ .../hadoop/hdfs/server/namenode/NameNode.java | 6 - ...rloadSchemeWithMountTableConfigInHDFS.java | 4 +- ...ileSystemOverloadSchemeWithHdfsScheme.java | 142 +- .../hdfs/TestDistributedFileSystem.java | 30 +- .../hdfs/TestViewDistributedFileSystem.java | 47 + ...TestViewDistributedFileSystemContract.java | 94 + ...ewDistributedFileSystemWithMountLinks.java | 64 + .../server/namenode/TestCacheDirectives.java | 27 +- .../TestCacheDirectivesWithViewDFS.java | 56 + 15 files changed, 2795 insertions(+), 90 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemContract.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemWithMountLinks.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectivesWithViewDFS.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java index 344048f0ceeb1..603454210644d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java @@ -45,5 +45,4 @@ public interface FsConstants { String FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN = "fs.viewfs.overload.scheme.target.%s.impl"; String VIEWFS_TYPE = "viewfs"; - String VIEWFSOS_TYPE = "viewfsOverloadScheme"; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 422e7337b57fb..003694f2e9918 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -599,9 +599,10 @@ protected InodeTree(final Configuration config, final String viewName, if (!gotMountTableEntry) { if (!initingUriAsFallbackOnNoMounts) { - throw new IOException( - "ViewFs: Cannot initialize: Empty Mount table in config for " - + "viewfs://" + mountTableName + "/"); + throw new IOException(new StringBuilder( + "ViewFs: Cannot initialize: Empty Mount table in config for ") + .append(theUri.getScheme()).append("://").append(mountTableName) + .append("/").toString()); } StringBuilder msg = new StringBuilder("Empty mount table detected for ").append(theUri) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index ad62f94ec6297..8c659d1b15ed6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -259,13 +259,14 @@ public String getScheme() { } /** - * Returns the ViewFileSystem type. - * @return viewfs + * Returns false as it does not support to add fallback link automatically on + * no mounts. */ - String getType() { - return FsConstants.VIEWFS_TYPE; + boolean supportAutoAddingFallbackOnNoMounts() { + return false; } + /** * Called after a new FileSystem instance is constructed. * @param theUri a uri whose authority section names the host, port, etc. for @@ -293,7 +294,7 @@ public void initialize(final URI theUri, final Configuration conf) try { myUri = new URI(getScheme(), authority, "/", null, null); boolean initingUriAsFallbackOnNoMounts = - !FsConstants.VIEWFS_TYPE.equals(getType()); + supportAutoAddingFallbackOnNoMounts(); fsState = new InodeTree(conf, tableName, myUri, initingUriAsFallbackOnNoMounts) { @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java index 2165a3f9ee688..5353e93b6f133 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java @@ -104,6 +104,7 @@ @InterfaceStability.Evolving public class ViewFileSystemOverloadScheme extends ViewFileSystem { private URI myUri; + private boolean supportAutoAddingFallbackOnNoMounts = true; public ViewFileSystemOverloadScheme() throws IOException { super(); } @@ -114,11 +115,19 @@ public String getScheme() { } /** - * Returns the ViewFileSystem type. - * @return viewfs + * By default returns false as ViewFileSystemOverloadScheme supports auto + * adding fallback on no mounts. */ - String getType() { - return FsConstants.VIEWFSOS_TYPE; + public boolean supportAutoAddingFallbackOnNoMounts() { + return this.supportAutoAddingFallbackOnNoMounts; + } + + /** + * Sets whether to add fallback automatically when no mount points found. + */ + public void setSupportAutoAddingFallbackOnNoMounts( + boolean addAutoFallbackOnNoMounts) { + this.supportAutoAddingFallbackOnNoMounts = addAutoFallbackOnNoMounts; } @Override @@ -287,4 +296,62 @@ public FileSystem getRawFileSystem(Path path, Configuration conf) } } + /** + * Gets the mount path info, which contains the target file system and + * remaining path to pass to the target file system. + */ + public MountPathInfo getMountPathInfo(Path path, + Configuration conf) throws IOException { + InodeTree.ResolveResult res; + try { + res = fsState.resolve(getUriPath(path), true); + FileSystem fs = res.isInternalDir() ? + (fsState.getRootFallbackLink() != null ? + ((ChRootedFileSystem) fsState + .getRootFallbackLink().targetFileSystem).getMyFs() : + fsGetter().get(path.toUri(), conf)) : + ((ChRootedFileSystem) res.targetFileSystem).getMyFs(); + return new MountPathInfo(res.remainingPath, res.resolvedPath, + fs); + } catch (FileNotFoundException e) { + // No link configured with passed path. + throw new NotInMountpointException(path, + "No link found for the given path."); + } + } + + /** + * A class to maintain the target file system and a path to pass to the target + * file system. + */ + public static class MountPathInfo { + private Path pathOnTarget; + private T targetFs; + + public MountPathInfo(Path pathOnTarget, String resolvedPath, T targetFs) { + this.pathOnTarget = pathOnTarget; + this.targetFs = targetFs; + } + + public Path getPathOnTarget() { + return this.pathOnTarget; + } + + public T getTargetFs() { + return this.targetFs; + } + } + + /** + * @return Gets the fallback file system configured. Usually, this will be the + * default cluster. + */ + public FileSystem getFallbackFileSystem() { + if (fsState.getRootFallbackLink() == null) { + return null; + } + return ((ChRootedFileSystem) fsState.getRootFallbackLink().targetFileSystem) + .getMyFs(); + } + } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 7694f789e1cf5..1d5c5af8c38e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -186,7 +186,7 @@ public void initialize(URI uri, Configuration conf) throws IOException { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } - this.dfs = new DFSClient(uri, conf, statistics); + initDFSClient(uri, conf); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); this.workingDir = getHomeDirectory(); @@ -200,6 +200,10 @@ public StorageStatistics provide() { }); } + void initDFSClient(URI theUri, Configuration conf) throws IOException { + this.dfs = new DFSClient(theUri, conf, statistics); + } + @Override public Path getWorkingDirectory() { return workingDir; @@ -1510,10 +1514,14 @@ protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) @Override public void close() throws IOException { try { - dfs.closeOutputStreams(false); + if (dfs != null) { + dfs.closeOutputStreams(false); + } super.close(); } finally { - dfs.close(); + if (dfs != null) { + dfs.close(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java new file mode 100644 index 0000000000000..0a681693a47b5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java @@ -0,0 +1,2307 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStoragePolicySpi; +import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.PartialListing; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.PathHandle; +import org.apache.hadoop.fs.QuotaUsage; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclStatus; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; +import org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; +import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolEntry; +import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.HdfsPathHandle; +import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; +import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; +import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.io.MultipleIOException; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.token.DelegationTokenIssuer; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.Progressable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +/** + * The ViewDistributedFileSystem is an extended class to DistributedFileSystem + * with additional mounting functionality. The goal is to have better API + * compatibility for HDFS users when using mounting + * filesystem(ViewFileSystemOverloadScheme). + * The ViewFileSystemOverloadScheme{@link ViewFileSystemOverloadScheme} is a new + * filesystem with inherited mounting functionality from ViewFileSystem. + * For the user who is using ViewFileSystemOverloadScheme by setting + * fs.hdfs.impl=org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme, now + * they can set fs.hdfs.impl=org.apache.hadoop.hdfs.ViewDistributedFileSystem. + * So, that the hdfs users will get closely compatible API with mount + * functionality. For the rest of all other schemes can continue to use + * ViewFileSystemOverloadScheme class directly for mount functionality. Please + * note that ViewFileSystemOverloadScheme provides only + * ViewFileSystem{@link ViewFileSystem} APIs. + * If user configured this class but no mount point configured? Then it will + * simply work as existing DistributedFileSystem class. If user configured both + * fs.hdfs.impl to this class and mount configurations, then users will be able + * to make calls the APIs available in this class, they are nothing but DFS + * APIs, but they will be delegated to viewfs functionality. Please note, APIs + * without any path in arguments( ex: isInSafeMode), will be delegated to + * default filesystem only, that is the configured fallback link. If you want to + * make these API calls on specific child filesystem, you may want to initialize + * them separately and call. In ViewDistributedFileSystem, we strongly recommend + * to configure linkFallBack when you add mount links and it's recommended to + * point be to your base cluster, usually your current fs.defaultFS if that's + * pointing to hdfs. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class ViewDistributedFileSystem extends DistributedFileSystem { + private static final Logger LOGGER = + LoggerFactory.getLogger(ViewDistributedFileSystem.class); + + // A mounting file system. + private ViewFileSystemOverloadScheme vfs; + // A default DFS, which should have set via linkFallback + private DistributedFileSystem defaultDFS; + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + try { + this.vfs = tryInitializeMountingViewFs(uri, conf); + } catch (IOException ioe) { + LOGGER.debug(new StringBuilder("Mount tree initialization failed with ") + .append("the reason => {}. Falling back to regular DFS") + .append(" initialization. Please re-initialize the fs after updating") + .append(" mount point.").toString(), ioe.getMessage()); + // Previous super.initialize would have skipped the dfsclient init and + // setWorkingDirectory as we planned to initialize vfs. Since vfs init + // failed, let's init dfsClient now. + super.initDFSClient(uri, conf); + super.setWorkingDirectory(super.getHomeDirectory()); + return; + } + + setConf(conf); + // A child DFS with the current initialized URI. This must be same as + // fallback fs. The fallback must point to root of your filesystems. + // Some APIs(without path in argument, for example isInSafeMode) will + // support only for base cluster filesystem. Only that APIs will use this + // fs. + defaultDFS = (DistributedFileSystem) this.vfs.getFallbackFileSystem(); + // Please don't access internal dfs client directly except in tests. + dfs = (defaultDFS != null) ? defaultDFS.dfs : null; + super.setWorkingDirectory(this.vfs.getHomeDirectory()); + } + + @Override + void initDFSClient(URI uri, Configuration conf) throws IOException { + // Since we plan to initialize vfs in this class, we will not need to + // initialize DFS client. + } + + public ViewDistributedFileSystem() { + } + + private ViewFileSystemOverloadScheme tryInitializeMountingViewFs(URI theUri, + Configuration conf) throws IOException { + ViewFileSystemOverloadScheme viewFs = new ViewFileSystemOverloadScheme(); + viewFs.setSupportAutoAddingFallbackOnNoMounts(false); + viewFs.initialize(theUri, conf); + return viewFs; + } + + @Override + public URI getUri() { + if (this.vfs == null) { + return super.getUri(); + } + return this.vfs.getUri(); + } + + @Override + public String getScheme() { + if (this.vfs == null) { + return super.getScheme(); + } + return this.vfs.getScheme(); + } + + @Override + public Path getWorkingDirectory() { + if (this.vfs == null) { + return super.getWorkingDirectory(); + } + return this.vfs.getWorkingDirectory(); + } + + @Override + public void setWorkingDirectory(Path dir) { + if (this.vfs == null) { + super.setWorkingDirectory(dir); + return; + } + this.vfs.setWorkingDirectory(dir); + } + + @Override + public Path getHomeDirectory() { + if (super.dfs == null) { + return null; + } + if (this.vfs == null) { + return super.getHomeDirectory(); + } + return this.vfs.getHomeDirectory(); + } + + /** + * Returns only default cluster getHedgedReadMetrics. + */ + @Override + public DFSHedgedReadMetrics getHedgedReadMetrics() { + if (this.vfs == null) { + return super.getHedgedReadMetrics(); + } + checkDefaultDFS(defaultDFS, "getHedgedReadMetrics"); + return defaultDFS.getHedgedReadMetrics(); + } + + @Override + public BlockLocation[] getFileBlockLocations(FileStatus fs, long start, + long len) throws IOException { + if (this.vfs == null) { + return super.getFileBlockLocations(fs, start, len); + } + return this.vfs.getFileBlockLocations(fs, start, len); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path p, final long start, + final long len) throws IOException { + if (this.vfs == null) { + return super.getFileBlockLocations(p, start, len); + } + return this.vfs.getFileBlockLocations(p, start, len); + } + + @Override + public void setVerifyChecksum(final boolean verifyChecksum) { + if (this.vfs == null) { + super.setVerifyChecksum(verifyChecksum); + return; + } + this.vfs.setVerifyChecksum(verifyChecksum); + } + + @Override + public boolean recoverLease(final Path f) throws IOException { + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "recoverLease"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .recoverLease(mountPathInfo.getPathOnTarget()); + } + + @Override + public FSDataInputStream open(final Path f, final int bufferSize) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.open(f, bufferSize); + } + + return this.vfs.open(f, bufferSize); + } + + @Override + public FSDataInputStream open(PathHandle fd, int bufferSize) + throws IOException { + return this.vfs.open(fd, bufferSize); + } + + @Override + protected HdfsPathHandle createPathHandle(FileStatus st, + Options.HandleOpt... opts) { + if (this.vfs == null) { + return super.createPathHandle(st, opts); + } + throw new UnsupportedOperationException(); + } + + @Override + public FSDataOutputStream append(final Path f, final int bufferSize, + final Progressable progress) throws IOException { + if (this.vfs == null) { + return super.append(f, bufferSize, progress); + } + return this.vfs.append(f, bufferSize, progress); + } + + @Override + public FSDataOutputStream append(Path f, final EnumSet flag, + final int bufferSize, final Progressable progress) throws IOException { + if (this.vfs == null) { + return super.append(f, flag, bufferSize, progress); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "append"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .append(mountPathInfo.getPathOnTarget(), flag, bufferSize, progress); + } + + @Override + public FSDataOutputStream append(Path f, final EnumSet flag, + final int bufferSize, final Progressable progress, + final InetSocketAddress[] favoredNodes) throws IOException { + if (this.vfs == null) { + return super.append(f, flag, bufferSize, progress, favoredNodes); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "append"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .append(mountPathInfo.getPathOnTarget(), flag, bufferSize, progress, + favoredNodes); + } + + @Override + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + if (this.vfs == null) { + return super + .create(f, permission, overwrite, bufferSize, replication, blockSize, + progress); + } + return this.vfs + .create(f, permission, overwrite, bufferSize, replication, blockSize, + progress); + } + + @Override + public HdfsDataOutputStream create(final Path f, + final FsPermission permission, final boolean overwrite, + final int bufferSize, final short replication, final long blockSize, + final Progressable progress, final InetSocketAddress[] favoredNodes) + throws IOException { + if (this.vfs == null) { + return super + .create(f, permission, overwrite, bufferSize, replication, blockSize, + progress, favoredNodes); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "create"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .create(mountPathInfo.getPathOnTarget(), permission, overwrite, + bufferSize, replication, blockSize, progress, favoredNodes); + } + + @Override + //DFS specific API + public FSDataOutputStream create(final Path f, final FsPermission permission, + final EnumSet cflags, final int bufferSize, + final short replication, final long blockSize, + final Progressable progress, final Options.ChecksumOpt checksumOpt) + throws IOException { + if (this.vfs == null) { + return super + .create(f, permission, cflags, bufferSize, replication, blockSize, + progress, checksumOpt); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "create"); + return mountPathInfo.getTargetFs() + .create(mountPathInfo.getPathOnTarget(), permission, cflags, bufferSize, + replication, blockSize, progress, checksumOpt); + } + + void checkDFS(FileSystem fs, String methodName) { + if (!(fs instanceof DistributedFileSystem)) { + String msg = new StringBuilder("This API:").append(methodName) + .append(" is specific to DFS. Can't run on other fs:") + .append(fs.getUri()).toString(); + throw new UnsupportedOperationException(msg); + } + } + + void checkDefaultDFS(FileSystem fs, String methodName) { + if (fs == null) { + String msg = new StringBuilder("This API:").append(methodName).append( + " cannot be supported without default cluster(that is linkFallBack).") + .toString(); + throw new UnsupportedOperationException(msg); + } + } + + @Override + // DFS specific API + protected HdfsDataOutputStream primitiveCreate(Path f, + FsPermission absolutePermission, EnumSet flag, int bufferSize, + short replication, long blockSize, Progressable progress, + Options.ChecksumOpt checksumOpt) throws IOException { + if (this.vfs == null) { + return super + .primitiveCreate(f, absolutePermission, flag, bufferSize, replication, + blockSize, progress, checksumOpt); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "primitiveCreate"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .primitiveCreate(f, absolutePermission, flag, bufferSize, replication, + blockSize, progress, checksumOpt); + } + + @Override + public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, + EnumSet flags, int bufferSize, short replication, + long blockSize, Progressable progress) throws IOException { + if (this.vfs == null) { + return super + .createNonRecursive(f, permission, flags, bufferSize, replication, + bufferSize, progress); + } + return this.vfs + .createNonRecursive(f, permission, flags, bufferSize, replication, + bufferSize, progress); + } + + @Override + public boolean setReplication(final Path f, final short replication) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.setReplication(f, replication); + } + return this.vfs.setReplication(f, replication); + } + + @Override + public void setStoragePolicy(Path src, String policyName) throws IOException { + if (this.vfs == null) { + super.setStoragePolicy(src, policyName); + return; + } + this.vfs.setStoragePolicy(src, policyName); + } + + @Override + public void unsetStoragePolicy(Path src) throws IOException { + if (this.vfs == null) { + super.unsetStoragePolicy(src); + return; + } + this.vfs.unsetStoragePolicy(src); + } + + @Override + public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException { + if (this.vfs == null) { + return super.getStoragePolicy(src); + } + return this.vfs.getStoragePolicy(src); + } + + @Override + public Collection getAllStoragePolicies() + throws IOException { + if (this.vfs == null) { + return super.getAllStoragePolicies(); + } + Collection allStoragePolicies = + this.vfs.getAllStoragePolicies(); + return (Collection) allStoragePolicies; + } + + @Override + public long getBytesWithFutureGenerationStamps() throws IOException { + if (this.vfs == null) { + return super.getBytesWithFutureGenerationStamps(); + } + checkDefaultDFS(defaultDFS, "getBytesWithFutureGenerationStamps"); + return defaultDFS.getBytesWithFutureGenerationStamps(); + } + + @Deprecated + @Override + public BlockStoragePolicy[] getStoragePolicies() throws IOException { + if (this.vfs == null) { + return super.getStoragePolicies(); + } + checkDefaultDFS(defaultDFS, "getStoragePolicies"); + return defaultDFS.getStoragePolicies(); + } + + @Override + //Make sure your target fs supports this API, otherwise you will get + // Unsupported operation exception. + public void concat(Path trg, Path[] psrcs) throws IOException { + if (this.vfs == null) { + super.concat(trg, psrcs); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(trg, getConf()); + mountPathInfo.getTargetFs().concat(mountPathInfo.getPathOnTarget(), psrcs); + } + + @SuppressWarnings("deprecation") + @Override + public boolean rename(final Path src, final Path dst) throws IOException { + if (this.vfs == null) { + return super.rename(src, dst); + } + return this.vfs.rename(src, dst); + } + + @SuppressWarnings("deprecation") + @Override + public void rename(Path src, Path dst, final Options.Rename... options) + throws IOException { + if (this.vfs == null) { + super.rename(src, dst, options); + return; + } + + // TODO: revisit + ViewFileSystemOverloadScheme.MountPathInfo mountSrcPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + checkDFS(mountSrcPathInfo.getTargetFs(), "rename"); + + ViewFileSystemOverloadScheme.MountPathInfo mountDstPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + checkDFS(mountDstPathInfo.getTargetFs(), "rename"); + + //Check both in same cluster. + if (!mountSrcPathInfo.getTargetFs().getUri() + .equals(mountDstPathInfo.getTargetFs().getUri())) { + throw new HadoopIllegalArgumentException( + "Can't rename across file systems."); + } + + ((DistributedFileSystem) mountSrcPathInfo.getTargetFs()) + .rename(mountSrcPathInfo.getPathOnTarget(), + mountDstPathInfo.getPathOnTarget(), options); + } + + @Override + public boolean truncate(final Path f, final long newLength) + throws IOException { + if (this.vfs == null) { + return super.truncate(f, newLength); + } + return this.vfs.truncate(f, newLength); + } + + public boolean delete(final Path f, final boolean recursive) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.delete(f, recursive); + } + return this.vfs.delete(f, recursive); + } + + @Override + public ContentSummary getContentSummary(Path f) throws IOException { + if (this.vfs == null) { + return super.getContentSummary(f); + } + return this.vfs.getContentSummary(f); + } + + @Override + public QuotaUsage getQuotaUsage(Path f) throws IOException { + if (this.vfs == null) { + return super.getQuotaUsage(f); + } + return this.vfs.getQuotaUsage(f); + } + + @Override + public void setQuota(Path src, final long namespaceQuota, + final long storagespaceQuota) throws IOException { + if (this.vfs == null) { + super.setQuota(src, namespaceQuota, storagespaceQuota); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + mountPathInfo.getTargetFs() + .setQuota(mountPathInfo.getPathOnTarget(), namespaceQuota, + storagespaceQuota); + } + + @Override + public void setQuotaByStorageType(Path src, final StorageType type, + final long quota) throws IOException { + if (this.vfs == null) { + super.setQuotaByStorageType(src, type, quota); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + mountPathInfo.getTargetFs() + .setQuotaByStorageType(mountPathInfo.getPathOnTarget(), type, quota); + } + + @Override + public FileStatus[] listStatus(Path p) throws IOException { + if (this.vfs == null) { + return super.listStatus(p); + } + return this.vfs.listStatus(p); + } + + @Override + public RemoteIterator listLocatedStatus(final Path f, + final PathFilter filter) throws FileNotFoundException, IOException { + if (this.vfs == null) { + return super.listLocatedStatus(f, filter); + } + return this.vfs.listLocatedStatus(f, filter); + } + + @Override + public RemoteIterator listStatusIterator(final Path p) + throws IOException { + if (this.vfs == null) { + return super.listStatusIterator(p); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(p, getConf()); + return mountPathInfo.getTargetFs() + .listStatusIterator(mountPathInfo.getPathOnTarget()); + } + + @Override + public RemoteIterator> batchedListStatusIterator( + final List paths) throws IOException { + if (this.vfs == null) { + return super.batchedListStatusIterator(paths); + } + // TODO: revisit for correct implementation. + return this.defaultDFS.batchedListStatusIterator(paths); + } + + @Override + public RemoteIterator> batchedListLocatedStatusIterator( + final List paths) throws IOException { + if (this.vfs == null) { + return super.batchedListLocatedStatusIterator(paths); + } + // TODO: revisit for correct implementation. + return this.defaultDFS.batchedListLocatedStatusIterator(paths); + } + + public boolean mkdir(Path f, FsPermission permission) throws IOException { + if (this.vfs == null) { + return super.mkdir(f, permission); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "mkdir"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .mkdir(mountPathInfo.getPathOnTarget(), permission); + } + + @Override + public boolean mkdirs(Path f, FsPermission permission) throws IOException { + if (this.vfs == null) { + return super.mkdirs(f, permission); + } + return this.vfs.mkdirs(f, permission); + } + + @SuppressWarnings("deprecation") + @Override + protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) + throws IOException { + if (this.vfs == null) { + return super.primitiveMkdir(f, absolutePermission); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "primitiveMkdir"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .primitiveMkdir(mountPathInfo.getPathOnTarget(), absolutePermission); + } + + @Override + public void close() throws IOException { + if (this.vfs != null) { + this.vfs.close(); + } + super.close(); + } + + @InterfaceAudience.Private + @Override + public DFSClient getClient() { + if (this.vfs == null) { + return super.getClient(); + } + checkDefaultDFS(defaultDFS, "getClient"); + return defaultDFS.getClient(); + } + + @Override + public FsStatus getStatus(Path p) throws IOException { + if (this.vfs == null) { + return super.getStatus(p); + } + return this.vfs.getStatus(p); + } + + @Override + public long getMissingBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getMissingBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getMissingBlocksCount"); + return defaultDFS.getMissingBlocksCount(); + } + + @Override + public long getPendingDeletionBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getPendingDeletionBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getPendingDeletionBlocksCount"); + return defaultDFS.getPendingDeletionBlocksCount(); + } + + @Override + public long getMissingReplOneBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getMissingReplOneBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getMissingReplOneBlocksCount"); + return defaultDFS.getMissingReplOneBlocksCount(); + } + + @Override + public long getLowRedundancyBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getLowRedundancyBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getLowRedundancyBlocksCount"); + return defaultDFS.getLowRedundancyBlocksCount(); + } + + @Override + public long getCorruptBlocksCount() throws IOException { + if (this.vfs == null) { + return super.getCorruptBlocksCount(); + } + checkDefaultDFS(defaultDFS, "getCorruptBlocksCount"); + return defaultDFS.getLowRedundancyBlocksCount(); + } + + @Override + public RemoteIterator listCorruptFileBlocks(final Path path) + throws IOException { + if (this.vfs == null) { + return super.listCorruptFileBlocks(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .listCorruptFileBlocks(mountPathInfo.getPathOnTarget()); + } + + @Override + public DatanodeInfo[] getDataNodeStats() throws IOException { + if (this.vfs == null) { + return super.getDataNodeStats(); + } + checkDefaultDFS(defaultDFS, "getDataNodeStats"); + return defaultDFS.getDataNodeStats(); + } + + @Override + public DatanodeInfo[] getDataNodeStats( + final HdfsConstants.DatanodeReportType type) throws IOException { + if (this.vfs == null) { + return super.getDataNodeStats(type); + } + checkDefaultDFS(defaultDFS, "getDataNodeStats"); + return defaultDFS.getDataNodeStats(type); + } + + @Override + public boolean setSafeMode(HdfsConstants.SafeModeAction action) + throws IOException { + if (this.vfs == null) { + return super.setSafeMode(action); + } + checkDefaultDFS(defaultDFS, "setSafeMode"); + return defaultDFS.setSafeMode(action); + } + + @Override + public boolean setSafeMode(HdfsConstants.SafeModeAction action, + boolean isChecked) throws IOException { + if (this.vfs == null) { + return super.setSafeMode(action, isChecked); + } + checkDefaultDFS(defaultDFS, "setSafeMode"); + return defaultDFS.setSafeMode(action, isChecked); + } + + @Override + public boolean saveNamespace(long timeWindow, long txGap) throws IOException { + if (this.vfs == null) { + return super.saveNamespace(timeWindow, txGap); + } + checkDefaultDFS(defaultDFS, "saveNamespace"); + return defaultDFS.saveNamespace(timeWindow, txGap); + } + + @Override + public void saveNamespace() throws IOException { + if (this.vfs == null) { + super.saveNamespace(); + return; + } + checkDefaultDFS(defaultDFS, "saveNamespace"); + defaultDFS.saveNamespace(); + } + + @Override + public long rollEdits() throws IOException { + if (this.vfs == null) { + return super.rollEdits(); + } + checkDefaultDFS(defaultDFS, "rollEdits"); + return defaultDFS.rollEdits(); + } + + @Override + public boolean restoreFailedStorage(String arg) throws IOException { + if (this.vfs == null) { + return super.restoreFailedStorage(arg); + } + checkDefaultDFS(defaultDFS, "restoreFailedStorage"); + return defaultDFS.restoreFailedStorage(arg); + } + + @Override + public void refreshNodes() throws IOException { + if (this.vfs == null) { + super.refreshNodes(); + return; + } + checkDefaultDFS(defaultDFS, "refreshNodes"); + defaultDFS.refreshNodes(); + } + + @Override + public void finalizeUpgrade() throws IOException { + if (this.vfs == null) { + super.finalizeUpgrade(); + return; + } + checkDefaultDFS(defaultDFS, "finalizeUpgrade"); + defaultDFS.finalizeUpgrade(); + } + + @Override + public boolean upgradeStatus() throws IOException { + if (this.vfs == null) { + return super.upgradeStatus(); + } + checkDefaultDFS(defaultDFS, "upgradeStatus"); + return defaultDFS.upgradeStatus(); + } + + @Override + public RollingUpgradeInfo rollingUpgrade( + HdfsConstants.RollingUpgradeAction action) throws IOException { + if (this.vfs == null) { + return super.rollingUpgrade(action); + } + checkDefaultDFS(defaultDFS, "rollingUpgrade"); + return defaultDFS.rollingUpgrade(action); + } + + @Override + public void metaSave(String pathname) throws IOException { + if (this.vfs == null) { + super.metaSave(pathname); + return; + } + checkDefaultDFS(defaultDFS, "metaSave"); + defaultDFS.metaSave(pathname); + } + + @Override + public FsServerDefaults getServerDefaults() throws IOException { + if (this.vfs == null) { + return super.getServerDefaults(); + } + checkDefaultDFS(defaultDFS, "getServerDefaults"); + //TODO: Need to revisit. + return defaultDFS.getServerDefaults(); + } + + @Override + public FileStatus getFileStatus(final Path f) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.getFileStatus(f); + } + return this.vfs.getFileStatus(f); + } + + @SuppressWarnings("deprecation") + @Override + public void createSymlink(final Path target, final Path link, + final boolean createParent) throws IOException { + // Regular DFS behavior + if (this.vfs == null) { + super.createSymlink(target, link, createParent); + return; + } + + throw new UnsupportedOperationException( + "createSymlink is not supported in ViewHDFS"); + } + + @Override + public boolean supportsSymlinks() { + if (this.vfs == null) { + return super.supportsSymlinks(); + } + // we can enabled later if we want to support symlinks. + return false; + } + + @Override + public FileStatus getFileLinkStatus(final Path f) throws IOException { + if (this.vfs == null) { + return super.getFileLinkStatus(f); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + return mountPathInfo.getTargetFs() + .getFileLinkStatus(mountPathInfo.getPathOnTarget()); + } + + @Override + public Path getLinkTarget(Path path) throws IOException { + if(this.vfs==null){ + return super.getLinkTarget(path); + } + return this.vfs.getLinkTarget(path); + } + + @Override + protected Path resolveLink(Path f) throws IOException { + if(this.vfs==null){ + return super.resolveLink(f); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(f, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "resolveLink"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .resolveLink(mountPathInfo.getPathOnTarget()); + } + + @Override + public FileChecksum getFileChecksum(final Path f) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.getFileChecksum(f); + } + return this.vfs.getFileChecksum(f); + } + + @Override + public void setPermission(final Path f, final FsPermission permission) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.setPermission(f, permission); + return; + } + this.vfs.setPermission(f, permission); + } + + @Override + public void setOwner(final Path f, final String username, + final String groupname) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.setOwner(f, username, groupname); + return; + } + this.vfs.setOwner(f, username, groupname); + } + + @Override + public void setTimes(final Path f, final long mtime, final long atime) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.setTimes(f, mtime, atime); + return; + } + this.vfs.setTimes(f, mtime, atime); + } + + @Override + // DFS specific API + protected int getDefaultPort() { + return super.getDefaultPort(); + } + + @Override + public Token getDelegationToken(String renewer) + throws IOException { + if (this.vfs == null) { + return super.getDelegationToken(renewer); + } + //Let applications call getDelegationTokenIssuers and get respective + // delegation tokens from child fs. + throw new UnsupportedOperationException(); + } + + @Override + public void setBalancerBandwidth(long bandwidth) throws IOException { + if (this.vfs == null) { + super.setBalancerBandwidth(bandwidth); + return; + } + checkDefaultDFS(defaultDFS, "setBalancerBandwidth"); + defaultDFS.setBalancerBandwidth(bandwidth); + } + + @Override + public String getCanonicalServiceName() { + if (this.vfs == null) { + return super.getCanonicalServiceName(); + } + checkDefaultDFS(defaultDFS, "getCanonicalServiceName"); + return defaultDFS.getCanonicalServiceName(); + } + + @Override + protected URI canonicalizeUri(URI uri) { + if (this.vfs == null) { + return super.canonicalizeUri(uri); + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = null; + try { + mountPathInfo = this.vfs.getMountPathInfo(new Path(uri), getConf()); + } catch (IOException e) { + LOGGER.warn("Failed to resolve the uri as mount path", e); + return null; + } + checkDFS(mountPathInfo.getTargetFs(), "canonicalizeUri"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .canonicalizeUri(uri); + } + + @Override + public boolean isInSafeMode() throws IOException { + if (this.vfs == null) { + return super.isInSafeMode(); + } + checkDefaultDFS(defaultDFS, "isInSafeMode"); + return defaultDFS.isInSafeMode(); + } + + @Override + // DFS specific API + public void allowSnapshot(Path path) throws IOException { + if (this.vfs == null) { + super.allowSnapshot(path); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "allowSnapshot"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .allowSnapshot(mountPathInfo.getPathOnTarget()); + } + + @Override + public void disallowSnapshot(final Path path) throws IOException { + if (this.vfs == null) { + super.disallowSnapshot(path); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "disallowSnapshot"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .disallowSnapshot(mountPathInfo.getPathOnTarget()); + } + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + if (this.vfs == null) { + return super.createSnapshot(path, snapshotName); + } + return this.vfs.createSnapshot(path, snapshotName); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + if (this.vfs == null) { + super.renameSnapshot(path, snapshotOldName, snapshotNewName); + return; + } + this.vfs.renameSnapshot(path, snapshotOldName, snapshotNewName); + } + + @Override + //Ony for HDFS users + public SnapshottableDirectoryStatus[] getSnapshottableDirListing() + throws IOException { + if (this.vfs == null) { + return super.getSnapshottableDirListing(); + } + checkDefaultDFS(defaultDFS, "getSnapshottableDirListing"); + return defaultDFS.getSnapshottableDirListing(); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + if (this.vfs == null) { + super.deleteSnapshot(path, snapshotName); + return; + } + this.vfs.deleteSnapshot(path, snapshotName); + } + + @Override + public RemoteIterator snapshotDiffReportListingRemoteIterator( + final Path snapshotDir, final String fromSnapshot, + final String toSnapshot) throws IOException { + if (this.vfs == null) { + return super + .snapshotDiffReportListingRemoteIterator(snapshotDir, fromSnapshot, + toSnapshot); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(snapshotDir, getConf()); + checkDFS(mountPathInfo.getTargetFs(), + "snapshotDiffReportListingRemoteIterator"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .snapshotDiffReportListingRemoteIterator( + mountPathInfo.getPathOnTarget(), fromSnapshot, toSnapshot); + } + + @Override + public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir, + final String fromSnapshot, final String toSnapshot) throws IOException { + if (this.vfs == null) { + return super.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(snapshotDir, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getSnapshotDiffReport"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getSnapshotDiffReport(mountPathInfo.getPathOnTarget(), fromSnapshot, + toSnapshot); + } + + @Override + public boolean isFileClosed(final Path src) throws IOException { + if (this.vfs == null) { + return super.isFileClosed(src); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(src, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "isFileClosed"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .isFileClosed(mountPathInfo.getPathOnTarget()); + } + + @Override + public long addCacheDirective(CacheDirectiveInfo info) throws IOException { + if (this.vfs == null) { + return super.addCacheDirective(info); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "addCacheDirective"); + + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .addCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build()); + } + + @Override + public long addCacheDirective(CacheDirectiveInfo info, + EnumSet flags) throws IOException { + if (this.vfs == null) { + return super.addCacheDirective(info, flags); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "addCacheDirective"); + + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .addCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build(), flags); + } + + @Override + public void modifyCacheDirective(CacheDirectiveInfo info) throws IOException { + if (this.vfs == null) { + super.modifyCacheDirective(info); + return; + } + if (info.getPath() != null) { + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "modifyCacheDirective"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .modifyCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build()); + return; + } + + // No path available in CacheDirectiveInfo, Let's shoot to all child fs. + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.modifyCacheDirective(info); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void modifyCacheDirective(CacheDirectiveInfo info, + EnumSet flags) throws IOException { + if (this.vfs == null) { + super.modifyCacheDirective(info, flags); + return; + } + if (info.getPath() != null) { + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(info.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "modifyCacheDirective"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .modifyCacheDirective(new CacheDirectiveInfo.Builder(info) + .setPath(mountPathInfo.getPathOnTarget()).build(), flags); + return; + } + // No path available in CacheDirectiveInfo, Let's shoot to all child fs. + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.modifyCacheDirective(info, flags); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void removeCacheDirective(long id) throws IOException { + if (this.vfs == null) { + super.removeCacheDirective(id); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.removeCacheDirective(id); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public RemoteIterator listCacheDirectives( + CacheDirectiveInfo filter) throws IOException { + if (this.vfs == null) { + return super.listCacheDirectives(filter); + } + + if (filter != null && filter.getPath() != null) { + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(filter.getPath(), getConf()); + checkDFS(mountPathInfo.getTargetFs(), "listCacheDirectives"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .listCacheDirectives(new CacheDirectiveInfo.Builder(filter) + .setPath(mountPathInfo.getPathOnTarget()).build()); + } + + // No path available in filter. Let's try to shoot to all child fs. + final List> iters = new ArrayList<>(); + for (FileSystem fs : getChildFileSystems()) { + if (fs instanceof DistributedFileSystem) { + iters.add(((DistributedFileSystem) fs).listCacheDirectives(filter)); + } + } + if (iters.size() == 0) { + throw new UnsupportedOperationException( + "No DFS found in child fs. This API can't be supported in non DFS"); + } + + return new RemoteIterator() { + int currIdx = 0; + RemoteIterator currIter = iters.get(currIdx++); + + @Override + public boolean hasNext() throws IOException { + if (currIter.hasNext()) { + return true; + } + while (currIdx < iters.size()) { + currIter = iters.get(currIdx++); + if (currIter.hasNext()) { + return true; + } + } + return false; + } + + @Override + public CacheDirectiveEntry next() throws IOException { + if (hasNext()) { + return currIter.next(); + } + throw new NoSuchElementException("No more elements"); + } + }; + } + + //Currently Cache pool APIs supported only in default cluster. + @Override + public void addCachePool(CachePoolInfo info) throws IOException { + if (this.vfs == null) { + super.addCachePool(info); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.addCachePool(info); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void modifyCachePool(CachePoolInfo info) throws IOException { + if (this.vfs == null) { + super.modifyCachePool(info); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.modifyCachePool(info); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void removeCachePool(String poolName) throws IOException { + if (this.vfs == null) { + super.removeCachePool(poolName); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.removeCachePool(poolName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public RemoteIterator listCachePools() throws IOException { + if (this.vfs == null) { + return super.listCachePools(); + } + + List childDFSs = new ArrayList<>(); + for (FileSystem fs : getChildFileSystems()) { + if (fs instanceof DistributedFileSystem) { + childDFSs.add((DistributedFileSystem) fs); + } + } + if (childDFSs.size() == 0) { + throw new UnsupportedOperationException( + "No DFS found in child fs. This API can't be supported in non DFS"); + } + return new RemoteIterator() { + int curDfsIdx = 0; + RemoteIterator currIter = + childDFSs.get(curDfsIdx++).listCachePools(); + + @Override + public boolean hasNext() throws IOException { + if (currIter.hasNext()) { + return true; + } + while (curDfsIdx < childDFSs.size()) { + currIter = childDFSs.get(curDfsIdx++).listCachePools(); + if (currIter.hasNext()) { + return true; + } + } + return false; + } + + @Override + public CachePoolEntry next() throws IOException { + if (hasNext()) { + return currIter.next(); + } + throw new java.util.NoSuchElementException("No more entries"); + } + }; + } + + @Override + public void modifyAclEntries(Path path, List aclSpec) + throws IOException { + if (this.vfs == null) { + super.modifyAclEntries(path, aclSpec); + return; + } + this.vfs.modifyAclEntries(path, aclSpec); + } + + @Override + public void removeAclEntries(Path path, List aclSpec) + throws IOException { + if (this.vfs == null) { + super.removeAclEntries(path, aclSpec); + return; + } + this.vfs.removeAclEntries(path, aclSpec); + } + + @Override + public void removeDefaultAcl(Path path) throws IOException { + if (this.vfs == null) { + super.removeDefaultAcl(path); + return; + } + this.vfs.removeDefaultAcl(path); + } + + @Override + public void removeAcl(Path path) throws IOException { + if (this.vfs == null) { + super.removeAcl(path); + return; + } + this.vfs.removeAcl(path); + } + + @Override + public void setAcl(Path path, List aclSpec) throws IOException { + if (this.vfs == null) { + super.setAcl(path, aclSpec); + return; + } + this.vfs.setAcl(path, aclSpec); + } + + @Override + public AclStatus getAclStatus(Path path) throws IOException { + if (this.vfs == null) { + return super.getAclStatus(path); + } + return this.vfs.getAclStatus(path); + } + + @Override + public void createEncryptionZone(final Path path, final String keyName) + throws IOException { + if (this.vfs == null) { + super.createEncryptionZone(path, keyName); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "createEncryptionZone"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .createEncryptionZone(mountPathInfo.getPathOnTarget(), keyName); + } + + @Override + public EncryptionZone getEZForPath(final Path path) throws IOException { + if (this.vfs == null) { + return super.getEZForPath(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getEZForPath"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getEZForPath(mountPathInfo.getPathOnTarget()); + } + + /** + * Returns the results from default DFS (fallback). If you want the results + * from specific clusters, please invoke them on child fs instance directly. + */ + @Override + public RemoteIterator listEncryptionZones() + throws IOException { + if (this.vfs == null) { + return super.listEncryptionZones(); + } + checkDefaultDFS(defaultDFS, "listEncryptionZones"); + return defaultDFS.listEncryptionZones(); + } + + @Override + public void reencryptEncryptionZone(final Path zone, + final HdfsConstants.ReencryptAction action) throws IOException { + if (this.vfs == null) { + super.reencryptEncryptionZone(zone, action); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(zone, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "reencryptEncryptionZone"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .reencryptEncryptionZone(mountPathInfo.getPathOnTarget(), action); + } + + /** + * Returns the results from default DFS (fallback). If you want the results + * from specific clusters, please invoke them on child fs instance directly. + */ + @Override + public RemoteIterator listReencryptionStatus() + throws IOException { + if (this.vfs == null) { + return super.listReencryptionStatus(); + } + checkDefaultDFS(defaultDFS, "listReencryptionStatus"); + return defaultDFS.listReencryptionStatus(); + } + + @Override + public FileEncryptionInfo getFileEncryptionInfo(final Path path) + throws IOException { + if (this.vfs == null) { + return super.getFileEncryptionInfo(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getFileEncryptionInfo"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getFileEncryptionInfo(mountPathInfo.getPathOnTarget()); + } + + @Override + public void provisionEZTrash(final Path path, + final FsPermission trashPermission) throws IOException { + if (this.vfs == null) { + super.provisionEZTrash(path, trashPermission); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "provisionEZTrash"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .provisionEZTrash(mountPathInfo.getPathOnTarget(), trashPermission); + } + + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet flag) throws IOException { + if (this.vfs == null) { + super.setXAttr(path, name, value, flag); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + mountPathInfo.getTargetFs() + .setXAttr(mountPathInfo.getPathOnTarget(), name, value, flag); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + if (this.vfs == null) { + return super.getXAttr(path, name); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .getXAttr(mountPathInfo.getPathOnTarget(), name); + } + + @Override + public Map getXAttrs(Path path) throws IOException { + if (this.vfs == null) { + return super.getXAttrs(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .getXAttrs(mountPathInfo.getPathOnTarget()); + } + + @Override + public Map getXAttrs(Path path, List names) + throws IOException { + if (this.vfs == null) { + return super.getXAttrs(path, names); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .getXAttrs(mountPathInfo.getPathOnTarget(), names); + } + + @Override + public List listXAttrs(Path path) throws IOException { + if (this.vfs == null) { + return super.listXAttrs(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + return mountPathInfo.getTargetFs() + .listXAttrs(mountPathInfo.getPathOnTarget()); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + if (this.vfs == null) { + super.removeXAttr(path, name); + return; + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + mountPathInfo.getTargetFs() + .removeXAttr(mountPathInfo.getPathOnTarget(), name); + } + + @Override + public void access(Path path, FsAction mode) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + super.access(path, mode); + return; + } + this.vfs.access(path, mode); + } + + @Override + public URI getKeyProviderUri() throws IOException { + if (this.vfs == null) { + return super.getKeyProviderUri(); + } + checkDefaultDFS(defaultDFS, "getKeyProviderUri"); + return defaultDFS.getKeyProviderUri(); + } + + @Override + public KeyProvider getKeyProvider() throws IOException { + if (this.vfs == null) { + return super.getKeyProvider(); + } + checkDefaultDFS(defaultDFS, "getKeyProvider"); + return defaultDFS.getKeyProvider(); + } + + @Override + public DelegationTokenIssuer[] getAdditionalTokenIssuers() + throws IOException { + if (this.vfs == null) { + return super.getChildFileSystems(); + } + + return this.vfs.getChildFileSystems(); + } + + @Override + public DFSInotifyEventInputStream getInotifyEventStream() throws IOException { + if (this.vfs == null) { + return super.getInotifyEventStream(); + } + checkDefaultDFS(defaultDFS, "getInotifyEventStream"); + return defaultDFS.getInotifyEventStream(); + } + + @Override + public DFSInotifyEventInputStream getInotifyEventStream(long lastReadTxid) + throws IOException { + if (this.vfs == null) { + return super.getInotifyEventStream(); + } + checkDefaultDFS(defaultDFS, "getInotifyEventStream"); + return defaultDFS.getInotifyEventStream(); + } + + @Override + // DFS only API. + public void setErasureCodingPolicy(final Path path, final String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.setErasureCodingPolicy(path, ecPolicyName); + return; + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "setErasureCodingPolicy"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .setErasureCodingPolicy(mountPathInfo.getPathOnTarget(), ecPolicyName); + } + + @Override + public void satisfyStoragePolicy(Path src) throws IOException { + if (this.vfs == null) { + super.satisfyStoragePolicy(src); + return; + } + this.vfs.satisfyStoragePolicy(src); + } + + @Override + public ErasureCodingPolicy getErasureCodingPolicy(final Path path) + throws IOException { + if (this.vfs == null) { + return super.getErasureCodingPolicy(path); + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "getErasureCodingPolicy"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .getErasureCodingPolicy(mountPathInfo.getPathOnTarget()); + } + + /** + * Gets all erasure coding policies from all available child file systems. + */ + @Override + public Collection getAllErasureCodingPolicies() + throws IOException { + if (this.vfs == null) { + return super.getAllErasureCodingPolicies(); + } + FileSystem[] childFss = getChildFileSystems(); + List results = new ArrayList<>(); + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : childFss) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + results.addAll(dfs.getAllErasureCodingPolicies()); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + return results; + } + + @Override + public Map getAllErasureCodingCodecs() throws IOException { + if (this.vfs == null) { + return super.getAllErasureCodingCodecs(); + } + FileSystem[] childFss = getChildFileSystems(); + Map results = new HashMap<>(); + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : childFss) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + results.putAll(dfs.getAllErasureCodingCodecs()); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + return results; + } + + @Override + public AddErasureCodingPolicyResponse[] addErasureCodingPolicies( + ErasureCodingPolicy[] policies) throws IOException { + if (this.vfs == null) { + return super.addErasureCodingPolicies(policies); + } + List failedExceptions = new ArrayList<>(); + List results = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + results.addAll(Arrays.asList(dfs.addErasureCodingPolicies(policies))); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + return results.toArray(new AddErasureCodingPolicyResponse[results.size()]); + } + + @Override + public void removeErasureCodingPolicy(String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.removeErasureCodingPolicy(ecPolicyName); + return; + } + + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.removeErasureCodingPolicy(ecPolicyName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void enableErasureCodingPolicy(String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.enableErasureCodingPolicy(ecPolicyName); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.enableErasureCodingPolicy(ecPolicyName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void disableErasureCodingPolicy(String ecPolicyName) + throws IOException { + if (this.vfs == null) { + super.disableErasureCodingPolicy(ecPolicyName); + return; + } + List failedExceptions = new ArrayList<>(); + boolean isDFSExistsInChilds = false; + + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + isDFSExistsInChilds = true; + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + dfs.disableErasureCodingPolicy(ecPolicyName); + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (!isDFSExistsInChilds) { + throw new UnsupportedOperationException( + "No DFS available in child file systems."); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + } + + @Override + public void unsetErasureCodingPolicy(final Path path) throws IOException { + if (this.vfs == null) { + super.unsetErasureCodingPolicy(path); + return; + } + + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(path, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "unsetErasureCodingPolicy"); + ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .unsetErasureCodingPolicy(mountPathInfo.getPathOnTarget()); + } + + @Override + public ECTopologyVerifierResult getECTopologyResultForPolicies( + final String... policyNames) throws IOException { + if (this.vfs == null) { + return super.getECTopologyResultForPolicies(policyNames); + } + + List failedExceptions = new ArrayList<>(); + ECTopologyVerifierResult result = null; + for (FileSystem fs : getChildFileSystems()) { + if (!(fs instanceof DistributedFileSystem)) { + continue; + } + DistributedFileSystem dfs = (DistributedFileSystem) fs; + try { + result = dfs.getECTopologyResultForPolicies(policyNames); + if (!result.isSupported()) { + // whenever we see negative result. + return result; + } + } catch (IOException ioe) { + failedExceptions.add(ioe); + } + } + if (result == null) { + throw new UnsupportedOperationException( + "No DFS available in child filesystems"); + } + if (failedExceptions.size() > 0) { + throw MultipleIOException.createIOException(failedExceptions); + } + // Let's just return the last one. + return result; + } + + @Override + public Path getTrashRoot(Path path) { + if (this.vfs == null) { + return super.getTrashRoot(path); + } + return this.vfs.getTrashRoot(path); + } + + @Override + public Collection getTrashRoots(boolean allUsers) { + if (this.vfs == null) { + return super.getTrashRoots(allUsers); + } + List trashRoots = new ArrayList<>(); + for (FileSystem fs : getChildFileSystems()) { + trashRoots.addAll(fs.getTrashRoots(allUsers)); + } + return trashRoots; + } + + // Just proovided the same implementation as default in dfs as thats just + // delegated to FileSystem parent class. + @Override + protected Path fixRelativePart(Path p) { + return super.fixRelativePart(p); + } + + Statistics getFsStatistics() { + if (this.vfs == null) { + return super.getFsStatistics(); + } + return statistics; + } + + DFSOpsCountStatistics getDFSOpsCountStatistics() { + if (this.vfs == null) { + return super.getDFSOpsCountStatistics(); + } + return defaultDFS.getDFSOpsCountStatistics(); + } + + @Override + // Works only for HDFS + public HdfsDataOutputStreamBuilder createFile(Path path) { + if (this.vfs == null) { + return super.createFile(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = null; + try { + mountPathInfo = this.vfs.getMountPathInfo(path, getConf()); + } catch (IOException e) { + // TODO: can we return null here? + return null; + } + checkDFS(mountPathInfo.getTargetFs(), "createFile"); + return (HdfsDataOutputStreamBuilder) mountPathInfo.getTargetFs() + .createFile(mountPathInfo.getPathOnTarget()); + } + + @Deprecated + @Override + public RemoteIterator listOpenFiles() throws IOException { + if (this.vfs == null) { + return super.listOpenFiles(); + } + checkDefaultDFS(defaultDFS, "listOpenFiles"); + return defaultDFS.listOpenFiles(); + } + + @Deprecated + @Override + public RemoteIterator listOpenFiles( + EnumSet openFilesTypes) + throws IOException { + if (this.vfs == null) { + return super.listOpenFiles(openFilesTypes); + } + checkDefaultDFS(defaultDFS, "listOpenFiles"); + return defaultDFS.listOpenFiles(openFilesTypes); + } + + @Override + public RemoteIterator listOpenFiles( + EnumSet openFilesTypes, String path) + throws IOException { + if (this.vfs == null) { + return super.listOpenFiles(openFilesTypes, path); + } + Path absF = fixRelativePart(new Path(path)); + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = + this.vfs.getMountPathInfo(absF, getConf()); + checkDFS(mountPathInfo.getTargetFs(), "listOpenFiles"); + return ((DistributedFileSystem) mountPathInfo.getTargetFs()) + .listOpenFiles(openFilesTypes, + mountPathInfo.getPathOnTarget().toString()); + } + + @Override + public HdfsDataOutputStreamBuilder appendFile(Path path) { + if (this.vfs == null) { + return super.appendFile(path); + } + ViewFileSystemOverloadScheme.MountPathInfo mountPathInfo = null; + try { + mountPathInfo = this.vfs.getMountPathInfo(path, getConf()); + } catch (IOException e) { + LOGGER.warn("Failed to resolve the path as mount path", e); + return null; + } + checkDFS(mountPathInfo.getTargetFs(), "appendFile"); + return (HdfsDataOutputStreamBuilder) mountPathInfo.getTargetFs() + .appendFile(mountPathInfo.getPathOnTarget()); + } + + @Override + public boolean hasPathCapability(Path path, String capability) + throws IOException { + if (this.vfs == null) { + return super.hasPathCapability(path, capability); + } + return this.vfs.hasPathCapability(path, capability); + } + + //Below API provided implementations are in ViewFS but not there in DFS. + @Override + public Path resolvePath(final Path f) throws IOException { + if (this.vfs == null) { + return super.resolvePath(f); + } + return this.vfs.resolvePath(f); + } + + @Override + @SuppressWarnings("deprecation") + public boolean delete(final Path f) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.delete(f); + } + return this.vfs.delete(f); + } + + @Override + public FileChecksum getFileChecksum(final Path f, final long length) + throws AccessControlException, FileNotFoundException, IOException { + if (this.vfs == null) { + return super.getFileChecksum(f, length); + } + return this.vfs.getFileChecksum(f, length); + } + + @Override + public boolean mkdirs(Path dir) throws IOException { + if (this.vfs == null) { + return super.mkdirs(dir); + } + return this.vfs.mkdirs(dir); + } + + @Override + public long getDefaultBlockSize(Path f) { + if (this.vfs == null) { + return super.getDefaultBlockSize(f); + } + return this.vfs.getDefaultBlockSize(f); + } + + @Override + public short getDefaultReplication(Path f) { + if (this.vfs == null) { + return super.getDefaultReplication(f); + } + return this.vfs.getDefaultReplication(f); + } + + @Override + public FsServerDefaults getServerDefaults(Path f) throws IOException { + if (this.vfs == null) { + return super.getServerDefaults(f); + } + return this.vfs.getServerDefaults(f); + } + + @Override + public void setWriteChecksum(final boolean writeChecksum) { + if (this.vfs == null) { + super.setWriteChecksum(writeChecksum); + return; + } + this.vfs.setWriteChecksum(writeChecksum); + } + + @Override + public FileSystem[] getChildFileSystems() { + if (this.vfs == null) { + return super.getChildFileSystems(); + } + return this.vfs.getChildFileSystems(); + } + + public ViewFileSystem.MountPoint[] getMountPoints() { + if (this.vfs == null) { + return null; + } + return this.vfs.getMountPoints(); + } + + @Override + public FsStatus getStatus() throws IOException { + if (this.vfs == null) { + return super.getStatus(); + } + return this.vfs.getStatus(); + } + + @Override + public long getUsed() throws IOException { + if (this.vfs == null) { + return super.getUsed(); + } + return this.vfs.getUsed(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 9bffaaabebeaf..e58072a7a7e41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; @@ -727,11 +726,6 @@ protected void initialize(Configuration conf) throws IOException { intervals); } } - // Currently NN uses FileSystem.get to initialize DFS in startTrashEmptier. - // If fs.hdfs.impl was overridden by core-site.xml, we may get other - // filesystem. To make sure we get DFS, we are setting fs.hdfs.impl to DFS. - // HDFS-15450 - conf.set(FS_HDFS_IMPL_KEY, DistributedFileSystem.class.getName()); UserGroupInformation.setConfiguration(conf); loginAsNameNodeUser(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFSOverloadSchemeWithMountTableConfigInHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFSOverloadSchemeWithMountTableConfigInHDFS.java index e7afbed0a1cd4..5e2f42b77a3a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFSOverloadSchemeWithMountTableConfigInHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFSOverloadSchemeWithMountTableConfigInHDFS.java @@ -37,8 +37,8 @@ public class TestViewFSOverloadSchemeWithMountTableConfigInHDFS @Before @Override - public void startCluster() throws IOException { - super.startCluster(); + public void setUp() throws IOException { + super.setUp(); String mountTableDir = URI.create(getConf().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)) .toString() + "/MountTable/"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java index 8b7eb88404a94..31674f8d1364e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java @@ -36,13 +36,15 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.test.PathUtils; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME; @@ -58,7 +60,7 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme { private static final String FS_IMPL_PATTERN_KEY = "fs.%s.impl"; private static final String HDFS_SCHEME = "hdfs"; private Configuration conf = null; - private MiniDFSCluster cluster = null; + private static MiniDFSCluster cluster = null; private URI defaultFSURI; private File localTargetDir; private static final String TEST_ROOT_DIR = PathUtils @@ -66,33 +68,52 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme { private static final String HDFS_USER_FOLDER = "/HDFSUser"; private static final String LOCAL_FOLDER = "/local"; + @BeforeClass + public static void init() throws IOException { + cluster = + new MiniDFSCluster.Builder(new Configuration()).numDataNodes(2).build(); + cluster.waitClusterUp(); + } + /** * Sets up the configurations and starts the MiniDFSCluster. */ @Before - public void startCluster() throws IOException { - conf = new Configuration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, - true); - conf.setInt( + public void setUp() throws IOException { + Configuration config = getNewConf(); + config.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); - conf.set(String.format(FS_IMPL_PATTERN_KEY, HDFS_SCHEME), + config.set(String.format(FS_IMPL_PATTERN_KEY, HDFS_SCHEME), ViewFileSystemOverloadScheme.class.getName()); - conf.set(String.format( - FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN, - HDFS_SCHEME), DistributedFileSystem.class.getName()); - conf.setBoolean(CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME, + config.setBoolean(CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME, CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitClusterUp(); + setConf(config); defaultFSURI = - URI.create(conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)); + URI.create(config.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)); localTargetDir = new File(TEST_ROOT_DIR, "/root/"); + localTargetDir.mkdirs(); Assert.assertEquals(HDFS_SCHEME, defaultFSURI.getScheme()); // hdfs scheme. } @After - public void tearDown() throws IOException { + public void cleanUp() throws IOException { + if (cluster != null) { + FileSystem fs = new DistributedFileSystem(); + fs.initialize(defaultFSURI, conf); + try { + FileStatus[] statuses = fs.listStatus(new Path("/")); + for (FileStatus st : statuses) { + Assert.assertTrue(fs.delete(st.getPath(), true)); + } + } finally { + fs.close(); + } + FileSystem.closeAll(); + } + } + + @AfterClass + public static void tearDown() throws IOException { if (cluster != null) { FileSystem.closeAll(); cluster.shutdown(); @@ -132,9 +153,9 @@ public void testMountLinkWithLocalAndHDFS() throws Exception { // /local/test Path localDir = new Path(LOCAL_FOLDER + "/test"); - try (ViewFileSystemOverloadScheme fs - = (ViewFileSystemOverloadScheme) FileSystem.get(conf)) { - Assert.assertEquals(2, fs.getMountPoints().length); + try (FileSystem fs + = FileSystem.get(conf)) { + Assert.assertEquals(2, fs.getChildFileSystems().length); fs.createNewFile(hdfsFile); // /HDFSUser/testfile fs.mkdirs(localDir); // /local/test } @@ -166,8 +187,13 @@ public void testMountLinkWithLocalAndHDFS() throws Exception { * hdfs://localhost:xxx/HDFSUser --> nonexistent://NonExistent/User/ * It should fail to add non existent fs link. */ - @Test(expected = IOException.class, timeout = 30000) + @Test(timeout = 30000) public void testMountLinkWithNonExistentLink() throws Exception { + testMountLinkWithNonExistentLink(true); + } + + public void testMountLinkWithNonExistentLink(boolean expectFsInitFailure) + throws Exception { final String userFolder = "/User"; final Path nonExistTargetPath = new Path("nonexistent://NonExistent" + userFolder); @@ -176,10 +202,17 @@ public void testMountLinkWithNonExistentLink() throws Exception { * Below addLink will create following mount points * hdfs://localhost:xxx/User --> nonexistent://NonExistent/User/ */ - addMountLinks(defaultFSURI.getAuthority(), new String[] {userFolder }, - new String[] {nonExistTargetPath.toUri().toString() }, conf); - FileSystem.get(conf); - Assert.fail("Expected to fail with non existent link"); + addMountLinks(defaultFSURI.getAuthority(), new String[] {userFolder}, + new String[] {nonExistTargetPath.toUri().toString()}, conf); + if (expectFsInitFailure) { + LambdaTestUtils.intercept(IOException.class, () -> { + FileSystem.get(conf); + }); + } else { + try (FileSystem fs = FileSystem.get(conf)) { + Assert.assertEquals("hdfs", fs.getScheme()); + } + } } /** @@ -271,14 +304,10 @@ public void testAccessViewFsPathWithoutAuthority() throws Exception { // check for viewfs path without authority Path viewFsRootPath = new Path("viewfs:/"); - try { - viewFsRootPath.getFileSystem(conf); - Assert.fail( - "Mount table with authority default should not be initialized"); - } catch (IOException e) { - assertTrue(e.getMessage().contains( - "Empty Mount table in config for viewfs://default/")); - } + LambdaTestUtils.intercept(IOException.class, + "Empty Mount table in config for viewfs://default", () -> { + viewFsRootPath.getFileSystem(conf); + }); // set the name of the default mount table here and // subsequent calls should succeed. @@ -334,18 +363,25 @@ public void testWithLinkFallBack() throws Exception { * * It cannot find any mount link. ViewFS expects a mount point from root. */ - @Test(expected = NotInMountpointException.class, timeout = 30000) - public void testCreateOnRootShouldFailWhenMountLinkConfigured() - throws Exception { + @Test(timeout = 30000) + public void testCreateOnRoot() throws Exception { + testCreateOnRoot(false); + } + + public void testCreateOnRoot(boolean fallbackExist) throws Exception { final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER); addMountLinks(defaultFSURI.getAuthority(), - new String[] {HDFS_USER_FOLDER, LOCAL_FOLDER }, + new String[] {HDFS_USER_FOLDER, LOCAL_FOLDER}, new String[] {hdfsTargetPath.toUri().toString(), - localTargetDir.toURI().toString() }, - conf); + localTargetDir.toURI().toString()}, conf); try (FileSystem fs = FileSystem.get(conf)) { - fs.createNewFile(new Path("/newFileOnRoot")); - Assert.fail("It should fail as root is read only in viewFS."); + if (fallbackExist) { + Assert.assertTrue(fs.createNewFile(new Path("/newFileOnRoot"))); + } else { + LambdaTestUtils.intercept(NotInMountpointException.class, () -> { + fs.createNewFile(new Path("/newFileOnRoot")); + }); + } } } @@ -433,15 +469,13 @@ public void testViewFsOverloadSchemeWithInnerCache() conf); // 1. Only 1 hdfs child file system should be there with cache. - try (ViewFileSystemOverloadScheme vfs = - (ViewFileSystemOverloadScheme) FileSystem.get(conf)) { + try (FileSystem vfs = FileSystem.get(conf)) { Assert.assertEquals(1, vfs.getChildFileSystems().length); } // 2. Two hdfs file systems should be there if no cache. conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); - try (ViewFileSystemOverloadScheme vfs = - (ViewFileSystemOverloadScheme) FileSystem.get(conf)) { + try (FileSystem vfs = FileSystem.get(conf)) { Assert.assertEquals(2, vfs.getChildFileSystems().length); } } @@ -466,8 +500,7 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets() conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); // Two hdfs file systems should be there if no cache. - try (ViewFileSystemOverloadScheme vfs = - (ViewFileSystemOverloadScheme) FileSystem.get(conf)) { + try (FileSystem vfs = FileSystem.get(conf)) { Assert.assertEquals(2, vfs.getChildFileSystems().length); } } @@ -494,8 +527,7 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets() // Only one local file system should be there if no InnerCache, but fs // cache should work. conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); - try (ViewFileSystemOverloadScheme vfs = - (ViewFileSystemOverloadScheme) FileSystem.get(conf)) { + try (FileSystem vfs = FileSystem.get(conf)) { Assert.assertEquals(1, vfs.getChildFileSystems().length); } } @@ -656,4 +688,18 @@ private void readString(final FileSystem nfly, final Path testFile, public Configuration getConf() { return this.conf; } + + /** + * @return configuration. + */ + public Configuration getNewConf() { + return new Configuration(cluster.getConfiguration(0)); + } + + /** + * sets configuration. + */ + public void setConf(Configuration config) { + conf = config; + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index c4355d50fbbce..d6293709045c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -142,7 +142,7 @@ public class TestDistributedFileSystem { private boolean noXmlDefaults = false; - private HdfsConfiguration getTestConfiguration() { + HdfsConfiguration getTestConfiguration() { HdfsConfiguration conf; if (noXmlDefaults) { conf = new HdfsConfiguration(false); @@ -813,7 +813,7 @@ public void testStatistics() throws IOException { @Test public void testStatistics2() throws IOException, NoSuchAlgorithmException { - HdfsConfiguration conf = new HdfsConfiguration(); + HdfsConfiguration conf = getTestConfiguration(); conf.set(DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY, StoragePolicySatisfierMode.EXTERNAL.toString()); File tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString()); @@ -1475,7 +1475,7 @@ public void testCreateWithCustomChecksum() throws Exception { @Test(timeout=60000) public void testFileCloseStatus() throws IOException { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getTestConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); DistributedFileSystem fs = cluster.getFileSystem(); try { @@ -1495,7 +1495,7 @@ public void testFileCloseStatus() throws IOException { @Test public void testCreateWithStoragePolicy() throws Throwable { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getTestConfiguration(); try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .storageTypes( new StorageType[] {StorageType.DISK, StorageType.ARCHIVE, @@ -1534,7 +1534,7 @@ public void testCreateWithStoragePolicy() throws Throwable { @Test(timeout=60000) public void testListFiles() throws IOException { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getTestConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { @@ -1557,7 +1557,7 @@ public void testListFiles() throws IOException { @Test public void testListStatusOfSnapshotDirs() throws IOException { - MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()) + MiniDFSCluster cluster = new MiniDFSCluster.Builder(getTestConfiguration()) .build(); try { DistributedFileSystem dfs = cluster.getFileSystem(); @@ -1577,7 +1577,7 @@ public void testListStatusOfSnapshotDirs() throws IOException { @Test(timeout=10000) public void testDFSClientPeerReadTimeout() throws IOException { final int timeout = 1000; - final Configuration conf = new HdfsConfiguration(); + final Configuration conf = getTestConfiguration(); conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, timeout); // only need cluster to create a dfs client to get a peer @@ -1611,7 +1611,7 @@ public void testDFSClientPeerReadTimeout() throws IOException { @Test(timeout=60000) public void testGetServerDefaults() throws IOException { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getTestConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { cluster.waitActive(); @@ -1626,7 +1626,7 @@ public void testGetServerDefaults() throws IOException { @Test(timeout=10000) public void testDFSClientPeerWriteTimeout() throws IOException { final int timeout = 1000; - final Configuration conf = new HdfsConfiguration(); + final Configuration conf = getTestConfiguration(); conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, timeout); // only need cluster to create a dfs client to get a peer @@ -1663,7 +1663,7 @@ public void testDFSClientPeerWriteTimeout() throws IOException { @Test(timeout = 30000) public void testTotalDfsUsed() throws Exception { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getTestConfiguration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); @@ -1850,7 +1850,7 @@ public void testDFSDataOutputStreamBuilderForAppend() throws IOException { @Test public void testSuperUserPrivilege() throws Exception { - HdfsConfiguration conf = new HdfsConfiguration(); + HdfsConfiguration conf = getTestConfiguration(); File tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString()); final Path jksPath = new Path(tmpDir.toString(), "test.jks"); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, @@ -1903,7 +1903,7 @@ public Void run() throws Exception { @Test public void testListingStoragePolicyNonSuperUser() throws Exception { - HdfsConfiguration conf = new HdfsConfiguration(); + HdfsConfiguration conf = getTestConfiguration(); try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) { cluster.waitActive(); final DistributedFileSystem dfs = cluster.getFileSystem(); @@ -2055,7 +2055,7 @@ public Object run() throws Exception { @Test public void testStorageFavouredNodes() throws IOException, InterruptedException, TimeoutException { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getTestConfiguration(); try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .storageTypes(new StorageType[] {StorageType.SSD, StorageType.DISK}) .numDataNodes(3).storagesPerDatanode(2).build()) { @@ -2080,7 +2080,7 @@ public void testStorageFavouredNodes() @Test public void testGetECTopologyResultForPolicies() throws Exception { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getTestConfiguration(); try (MiniDFSCluster cluster = DFSTestUtil.setupCluster(conf, 9, 3, 0)) { DistributedFileSystem dfs = cluster.getFileSystem(); dfs.enableErasureCodingPolicy("RS-6-3-1024k"); @@ -2111,7 +2111,7 @@ public void testGetECTopologyResultForPolicies() throws Exception { @Test public void testECCloseCommittedBlock() throws Exception { - HdfsConfiguration conf = new HdfsConfiguration(); + HdfsConfiguration conf = getTestConfiguration(); conf.setInt(DFS_NAMENODE_FILE_CLOSE_NUM_COMMITTED_ALLOWED_KEY, 1); try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(3).build()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java new file mode 100644 index 0000000000000..3c5a0be303d4e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.test.Whitebox; + +import java.io.IOException; + +public class TestViewDistributedFileSystem extends TestDistributedFileSystem{ + @Override + HdfsConfiguration getTestConfiguration() { + HdfsConfiguration conf = super.getTestConfiguration(); + conf.set("fs.hdfs.impl", ViewDistributedFileSystem.class.getName()); + return conf; + } + + @Override + public void testStatistics() throws IOException { + FileSystem.getStatistics(HdfsConstants.HDFS_URI_SCHEME, + ViewDistributedFileSystem.class).reset(); + @SuppressWarnings("unchecked") + ThreadLocal data = + (ThreadLocal) Whitebox + .getInternalState(FileSystem + .getStatistics(HdfsConstants.HDFS_URI_SCHEME, + ViewDistributedFileSystem.class), "threadData"); + data.set(null); + super.testStatistics(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemContract.java new file mode 100644 index 0000000000000..418c814c8863b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemContract.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemContractBaseTest; +import org.apache.hadoop.fs.viewfs.ConfigUtil; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URI; + +public class TestViewDistributedFileSystemContract + extends TestHDFSFileSystemContract { + private static MiniDFSCluster cluster; + private static String defaultWorkingDirectory; + private static Configuration conf = new HdfsConfiguration(); + + @BeforeClass + public static void init() throws IOException { + final File basedir = GenericTestUtils.getRandomizedTestDir(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, + FileSystemContractBaseTest.TEST_UMASK); + cluster = new MiniDFSCluster.Builder(conf, basedir) + .numDataNodes(2) + .build(); + defaultWorkingDirectory = + "/user/" + UserGroupInformation.getCurrentUser().getShortUserName(); + } + + @Before + public void setUp() throws Exception { + conf.set("fs.hdfs.impl", ViewDistributedFileSystem.class.getName()); + URI defaultFSURI = + URI.create(conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)); + ConfigUtil.addLink(conf, defaultFSURI.getHost(), "/user", + defaultFSURI); + ConfigUtil.addLinkFallback(conf, defaultFSURI.getHost(), + defaultFSURI); + fs = FileSystem.get(conf); + } + + @AfterClass + public static void tearDownAfter() throws Exception { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Override + protected String getDefaultWorkingDirectory() { + return defaultWorkingDirectory; + } + + @Test + public void testRenameRootDirForbidden() throws Exception { + LambdaTestUtils.intercept(AccessControlException.class, + "InternalDir of ViewFileSystem is readonly", () -> { + super.testRenameRootDirForbidden(); + }); + } + + @Ignore("Ignore this test until HDFS-15532") + @Override + public void testLSRootDir() throws Throwable { + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemWithMountLinks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemWithMountLinks.java new file mode 100644 index 0000000000000..23b5793eee671 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystemWithMountLinks.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ConfigUtil; +import org.apache.hadoop.fs.viewfs.TestViewFileSystemOverloadSchemeWithHdfsScheme; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; + +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT; + +public class TestViewDistributedFileSystemWithMountLinks extends + TestViewFileSystemOverloadSchemeWithHdfsScheme { + @Override + public void setUp() throws IOException { + super.setUp(); + Configuration conf = getConf(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, + true); + conf.setInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); + conf.set("fs.hdfs.impl", + ViewDistributedFileSystem.class.getName()); + conf.setBoolean(CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME, + CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT); + URI defaultFSURI = + URI.create(conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)); + ConfigUtil.addLinkFallback(conf, defaultFSURI.getAuthority(), + new Path(defaultFSURI.toString()).toUri()); + setConf(conf); + } + + @Test(timeout = 30000) + public void testCreateOnRoot() throws Exception { + testCreateOnRoot(true); + } + + @Test(timeout = 30000) + public void testMountLinkWithNonExistentLink() throws Exception { + testMountLinkWithNonExistentLink(false); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index c5bc211977b54..9a525481407ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -132,17 +132,23 @@ private static HdfsConfiguration createCachingConf() { conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES, 2); conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES, 2); - return conf; } + /** + * @return the configuration. + */ + Configuration getConf() { + return this.conf; + } + @Before public void setup() throws Exception { conf = createCachingConf(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); cluster.waitActive(); - dfs = cluster.getFileSystem(); + dfs = getDFS(); proto = cluster.getNameNodeRpc(); namenode = cluster.getNameNode(); prevCacheManipulator = NativeIO.POSIX.getCacheManipulator(); @@ -150,6 +156,13 @@ public void setup() throws Exception { BlockReaderTestUtil.enableHdfsCachingTracing(); } + /** + * @return the dfs instance. + */ + DistributedFileSystem getDFS() throws IOException { + return (DistributedFileSystem) FileSystem.get(conf); + } + @After public void teardown() throws Exception { // Remove cache directives left behind by tests so that we release mmaps. @@ -1613,6 +1626,14 @@ public void testAddingCacheDirectiveInfosWhenCachingIsDisabled() "testAddingCacheDirectiveInfosWhenCachingIsDisabled:2"); } + /** + * @return the dfs instance for nnIdx. + */ + DistributedFileSystem getDFS(MiniDFSCluster cluster, int nnIdx) + throws IOException { + return cluster.getFileSystem(0); + } + @Test(timeout=120000) public void testExpiryTimeConsistency() throws Exception { conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1); @@ -1623,7 +1644,7 @@ public void testExpiryTimeConsistency() throws Exception { .build(); dfsCluster.transitionToActive(0); - DistributedFileSystem fs = dfsCluster.getFileSystem(0); + DistributedFileSystem fs = getDFS(dfsCluster, 0); final NameNode ann = dfsCluster.getNameNode(0); final Path filename = new Path("/file"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectivesWithViewDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectivesWithViewDFS.java new file mode 100644 index 0000000000000..b80c2a656c6e2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectivesWithViewDFS.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ConfigUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.ViewDistributedFileSystem; + +import java.io.IOException; +import java.net.URI; + +public class TestCacheDirectivesWithViewDFS extends TestCacheDirectives { + + @Override + public DistributedFileSystem getDFS() throws IOException { + Configuration conf = getConf(); + conf.set("fs.hdfs.impl", ViewDistributedFileSystem.class.getName()); + URI defaultFSURI = + URI.create(conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)); + ConfigUtil.addLinkFallback(conf, defaultFSURI.getHost(), + new Path(defaultFSURI.toString()).toUri()); + ConfigUtil.addLink(conf, defaultFSURI.getHost(), "/tmp", + new Path(defaultFSURI.toString()).toUri()); + return super.getDFS(); + } + + @Override + public DistributedFileSystem getDFS(MiniDFSCluster cluster, int nnIdx) + throws IOException { + Configuration conf = cluster.getConfiguration(nnIdx); + conf.set("fs.hdfs.impl", ViewDistributedFileSystem.class.getName()); + URI uri = cluster.getURI(0); + ConfigUtil.addLinkFallback(conf, uri.getHost(), uri); + ConfigUtil.addLink(conf, uri.getHost(), "/tmp", uri); + return cluster.getFileSystem(0); + } +} From 9b9f7ea16a299911fc75738401cb5a0f8713ead7 Mon Sep 17 00:00:00 2001 From: Zhengbo Li Date: Wed, 19 Aug 2020 13:14:50 -0400 Subject: [PATCH 085/335] YARN-10399 Refactor NodeQueueLoadMonitor class to make it extendable (#2228) Refactor NodeQueueLoadMonitor class to make it extendable --- .../scheduler/distributed/ClusterNode.java | 93 ++++++++ .../distributed/NodeQueueLoadMonitor.java | 217 +++++++++--------- .../distributed/QueueLimitCalculator.java | 1 - 3 files changed, 205 insertions(+), 106 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java new file mode 100644 index 0000000000000..f92a92bb17c40 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed; + +import java.util.Collection; +import java.util.HashSet; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.yarn.api.records.NodeId; + +/** + * Represents a node in the cluster from the NodeQueueLoadMonitor's perspective + */ +public class ClusterNode { + private final AtomicInteger queueLength = new AtomicInteger(0); + private final AtomicInteger queueWaitTime = new AtomicInteger(-1); + private long timestamp; + final NodeId nodeId; + private int queueCapacity = 0; + private final HashSet labels; + + public ClusterNode(NodeId nodeId) { + this.nodeId = nodeId; + this.labels = new HashSet<>(); + updateTimestamp(); + } + + public ClusterNode setQueueLength(int qLength) { + this.queueLength.set(qLength); + return this; + } + + public ClusterNode setQueueWaitTime(int wTime) { + this.queueWaitTime.set(wTime); + return this; + } + + public ClusterNode updateTimestamp() { + this.timestamp = System.currentTimeMillis(); + return this; + } + + public ClusterNode setQueueCapacity(int capacity) { + this.queueCapacity = capacity; + return this; + } + + public ClusterNode setNodeLabels(Collection labelsToAdd) { + labels.clear(); + labels.addAll(labelsToAdd); + return this; + } + + public boolean hasLabel(String label) { + return this.labels.contains(label); + } + + public long getTimestamp() { + return this.timestamp; + } + + public AtomicInteger getQueueLength() { + return this.queueLength; + } + + public AtomicInteger getQueueWaitTime() { + return this.queueWaitTime; + } + + public int getQueueCapacity() { + return this.queueCapacity; + } + + public boolean isQueueFull() { + return this.queueCapacity > 0 && + this.queueLength.get() >= this.queueCapacity; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java index fa2ba3060e333..bad98c54b5270 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java @@ -40,8 +40,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED; @@ -53,10 +53,10 @@ */ public class NodeQueueLoadMonitor implements ClusterMonitor { - private final static Logger LOG = LoggerFactory. + protected final static Logger LOG = LoggerFactory. getLogger(NodeQueueLoadMonitor.class); - private int numNodesForAnyAllocation = + protected int numNodesForAnyAllocation = DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED; /** @@ -70,14 +70,14 @@ public enum LoadComparator implements Comparator { @Override public int compare(ClusterNode o1, ClusterNode o2) { if (getMetric(o1) == getMetric(o2)) { - return (int)(o2.timestamp - o1.timestamp); + return (int)(o2.getTimestamp() - o1.getTimestamp()); } return getMetric(o1) - getMetric(o2); } public int getMetric(ClusterNode c) { return (this == QUEUE_LENGTH) ? - c.queueLength.get() : c.queueWaitTime.get(); + c.getQueueLength().get() : c.getQueueWaitTime().get(); } /** @@ -88,11 +88,11 @@ public int getMetric(ClusterNode c) { */ public boolean compareAndIncrement(ClusterNode c, int incrementSize) { if(this == QUEUE_LENGTH) { - int ret = c.queueLength.addAndGet(incrementSize); - if (ret <= c.queueCapacity) { + int ret = c.getQueueLength().addAndGet(incrementSize); + if (ret <= c.getQueueCapacity()) { return true; } - c.queueLength.addAndGet(-incrementSize); + c.getQueueLength().addAndGet(-incrementSize); return false; } // for queue wait time, we don't have any threshold. @@ -100,57 +100,19 @@ public boolean compareAndIncrement(ClusterNode c, int incrementSize) { } } - static class ClusterNode { - private AtomicInteger queueLength = new AtomicInteger(0); - private AtomicInteger queueWaitTime = new AtomicInteger(-1); - private long timestamp; - final NodeId nodeId; - private int queueCapacity = 0; - - public ClusterNode(NodeId nodeId) { - this.nodeId = nodeId; - updateTimestamp(); - } - - public ClusterNode setQueueLength(int qLength) { - this.queueLength.set(qLength); - return this; - } - - public ClusterNode setQueueWaitTime(int wTime) { - this.queueWaitTime.set(wTime); - return this; - } - - public ClusterNode updateTimestamp() { - this.timestamp = System.currentTimeMillis(); - return this; - } - - public ClusterNode setQueueCapacity(int capacity) { - this.queueCapacity = capacity; - return this; - } - - public boolean isQueueFull() { - return this.queueCapacity > 0 && - this.queueLength.get() >= this.queueCapacity; - } - } - private final ScheduledExecutorService scheduledExecutor; - private final List sortedNodes; - private final Map clusterNodes = + protected final List sortedNodes; + protected final Map clusterNodes = new ConcurrentHashMap<>(); - private final Map nodeByHostName = + protected final Map nodeByHostName = new ConcurrentHashMap<>(); - private final Map> nodeIdsByRack = + protected final Map> nodeIdsByRack = new ConcurrentHashMap<>(); - private final LoadComparator comparator; - private QueueLimitCalculator thresholdCalculator; - private ReentrantReadWriteLock sortedNodesLock = new ReentrantReadWriteLock(); - private ReentrantReadWriteLock clusterNodesLock = + protected final LoadComparator comparator; + protected QueueLimitCalculator thresholdCalculator; + protected ReentrantReadWriteLock sortedNodesLock = new ReentrantReadWriteLock(); + protected ReentrantReadWriteLock clusterNodesLock = new ReentrantReadWriteLock(); Runnable computeTask = new Runnable() { @@ -160,9 +122,7 @@ public void run() { writeLock.lock(); try { try { - List nodeIds = sortNodes(); - sortedNodes.clear(); - sortedNodes.addAll(nodeIds); + updateSortedNodes(); } catch (Exception ex) { LOG.warn("Got Exception while sorting nodes..", ex); } @@ -193,6 +153,14 @@ public NodeQueueLoadMonitor(long nodeComputationInterval, numNodesForAnyAllocation = numNodes; } + protected void updateSortedNodes() { + List nodeIds = sortNodes(true).stream() + .map(n -> n.nodeId) + .collect(Collectors.toList()); + sortedNodes.clear(); + sortedNodes.addAll(nodeIds); + } + List getSortedNodes() { return sortedNodes; } @@ -239,6 +207,7 @@ public void removeNode(RMNode removedRMNode) { ClusterNode node; try { node = this.clusterNodes.remove(removedRMNode.getNodeID()); + onNodeRemoved(node); } finally { writeLock.unlock(); } @@ -251,6 +220,13 @@ public void removeNode(RMNode removedRMNode) { } } + /** + * Provide an integration point for extended class + * @param node the node removed + */ + protected void onNodeRemoved(ClusterNode node) { + } + @Override public void updateNode(RMNode rmNode) { LOG.debug("Node update event from: {}", rmNode.getNodeID()); @@ -260,55 +236,80 @@ public void updateNode(RMNode rmNode) { opportunisticContainersStatus = OpportunisticContainersStatus.newInstance(); } - int opportQueueCapacity = - opportunisticContainersStatus.getOpportQueueCapacity(); - int estimatedQueueWaitTime = - opportunisticContainersStatus.getEstimatedQueueWaitTime(); - int waitQueueLength = opportunisticContainersStatus.getWaitQueueLength(); + // Add nodes to clusterNodes. If estimatedQueueTime is -1, ignore node // UNLESS comparator is based on queue length. ReentrantReadWriteLock.WriteLock writeLock = clusterNodesLock.writeLock(); writeLock.lock(); try { - ClusterNode currentNode = this.clusterNodes.get(rmNode.getNodeID()); - if (currentNode == null) { - if (rmNode.getState() != NodeState.DECOMMISSIONING && - (estimatedQueueWaitTime != -1 || - comparator == LoadComparator.QUEUE_LENGTH)) { - this.clusterNodes.put(rmNode.getNodeID(), - new ClusterNode(rmNode.getNodeID()) - .setQueueWaitTime(estimatedQueueWaitTime) - .setQueueLength(waitQueueLength) - .setQueueCapacity(opportQueueCapacity)); - LOG.info("Inserting ClusterNode [" + rmNode.getNodeID() + "] " + - "with queue wait time [" + estimatedQueueWaitTime + "] and " + - "wait queue length [" + waitQueueLength + "]"); - } else { - LOG.warn("IGNORING ClusterNode [" + rmNode.getNodeID() + "] " + - "with queue wait time [" + estimatedQueueWaitTime + "] and " + - "wait queue length [" + waitQueueLength + "]"); - } + ClusterNode clusterNode = this.clusterNodes.get(rmNode.getNodeID()); + if (clusterNode == null) { + onNewNodeAdded(rmNode, opportunisticContainersStatus); } else { - if (rmNode.getState() != NodeState.DECOMMISSIONING && - (estimatedQueueWaitTime != -1 || - comparator == LoadComparator.QUEUE_LENGTH)) { - currentNode + onExistingNodeUpdated(rmNode, clusterNode, opportunisticContainersStatus); + } + } finally { + writeLock.unlock(); + } + } + + protected void onNewNodeAdded( + RMNode rmNode, OpportunisticContainersStatus status) { + int opportQueueCapacity = status.getOpportQueueCapacity(); + int estimatedQueueWaitTime = status.getEstimatedQueueWaitTime(); + int waitQueueLength = status.getWaitQueueLength(); + + if (rmNode.getState() != NodeState.DECOMMISSIONING && + (estimatedQueueWaitTime != -1 || + comparator == LoadComparator.QUEUE_LENGTH)) { + this.clusterNodes.put(rmNode.getNodeID(), + new ClusterNode(rmNode.getNodeID()) .setQueueWaitTime(estimatedQueueWaitTime) .setQueueLength(waitQueueLength) - .updateTimestamp(); - LOG.debug("Updating ClusterNode [{}] with queue wait time [{}] and" + .setNodeLabels(rmNode.getNodeLabels()) + .setQueueCapacity(opportQueueCapacity)); + LOG.info( + "Inserting ClusterNode [{}] with queue wait time [{}] and " + + "wait queue length [{}]", + rmNode.getNode(), + estimatedQueueWaitTime, + waitQueueLength + ); + } else { + LOG.warn( + "IGNORING ClusterNode [{}] with queue wait time [{}] and " + + "wait queue length [{}]", + rmNode.getNode(), + estimatedQueueWaitTime, + waitQueueLength + ); + } + } + + protected void onExistingNodeUpdated( + RMNode rmNode, ClusterNode clusterNode, + OpportunisticContainersStatus status) { + + int estimatedQueueWaitTime = status.getEstimatedQueueWaitTime(); + int waitQueueLength = status.getWaitQueueLength(); + + if (rmNode.getState() != NodeState.DECOMMISSIONING && + (estimatedQueueWaitTime != -1 || + comparator == LoadComparator.QUEUE_LENGTH)) { + clusterNode + .setQueueWaitTime(estimatedQueueWaitTime) + .setQueueLength(waitQueueLength) + .setNodeLabels(rmNode.getNodeLabels()) + .updateTimestamp(); + LOG.debug("Updating ClusterNode [{}] with queue wait time [{}] and" + " wait queue length [{}]", rmNode.getNodeID(), - estimatedQueueWaitTime, waitQueueLength); + estimatedQueueWaitTime, waitQueueLength); - } else { - this.clusterNodes.remove(rmNode.getNodeID()); - LOG.info("Deleting ClusterNode [" + rmNode.getNodeID() + "] " + - "with queue wait time [" + currentNode.queueWaitTime + "] and " + - "wait queue length [" + currentNode.queueLength + "]"); - } - } - } finally { - writeLock.unlock(); + } else { + this.clusterNodes.remove(rmNode.getNodeID()); + LOG.info("Deleting ClusterNode [" + rmNode.getNodeID() + "] " + + "with queue wait time [" + clusterNode.getQueueWaitTime() + "] and " + + "wait queue length [" + clusterNode.getQueueLength() + "]"); } } @@ -374,7 +375,7 @@ public RMNode selectRackLocalNode(String rackName, Set blacklist) { } public RMNode selectAnyNode(Set blacklist) { - List nodeIds = selectLeastLoadedNodes(numNodesForAnyAllocation); + List nodeIds = getCandidatesForSelectAnyNode(); int size = nodeIds.size(); if (size <= 0) { return null; @@ -395,22 +396,26 @@ public RMNode selectAnyNode(Set blacklist) { return null; } - private void removeFromNodeIdsByRack(RMNode removedNode) { + protected List getCandidatesForSelectAnyNode() { + return selectLeastLoadedNodes(numNodesForAnyAllocation); + } + + protected void removeFromNodeIdsByRack(RMNode removedNode) { nodeIdsByRack.computeIfPresent(removedNode.getRackName(), (k, v) -> v).remove(removedNode.getNodeID()); } - private void addIntoNodeIdsByRack(RMNode addedNode) { + protected void addIntoNodeIdsByRack(RMNode addedNode) { nodeIdsByRack.compute(addedNode.getRackName(), (k, v) -> v == null ? ConcurrentHashMap.newKeySet() : v).add(addedNode.getNodeID()); } - private List sortNodes() { + protected List sortNodes(boolean excludeFullNodes) { ReentrantReadWriteLock.ReadLock readLock = clusterNodesLock.readLock(); readLock.lock(); try { - ArrayList aList = new ArrayList<>(this.clusterNodes.values()); - List retList = new ArrayList<>(); + ArrayList aList = new ArrayList<>(this.clusterNodes.values()); + List retList = new ArrayList<>(); Object[] nodes = aList.toArray(); // Collections.sort would do something similar by calling Arrays.sort // internally but would finally iterate through the input list (aList) @@ -420,9 +425,11 @@ private List sortNodes() { Arrays.sort(nodes, (Comparator)comparator); for (int j=0; j < nodes.length; j++) { ClusterNode cNode = (ClusterNode)nodes[j]; - // Exclude nodes whose queue is already full. - if (!cNode.isQueueFull()) { - retList.add(cNode.nodeId); + // Only add node to the result list when either condition is met: + // 1. we don't exclude full nodes + // 2. we do exclude full nodes, but the current node is not full + if (!excludeFullNodes || !cNode.isQueueFull()) { + retList.add(cNode); } } return retList; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/QueueLimitCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/QueueLimitCalculator.java index ab3a577d9f89f..8bd2f24d8b87c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/QueueLimitCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/QueueLimitCalculator.java @@ -20,7 +20,6 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.NodeQueueLoadMonitor.ClusterNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.NodeQueueLoadMonitor.LoadComparator; import java.util.List; From 4813a37023a80e927532857de65fcfdd0e25278c Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Thu, 20 Aug 2020 23:54:35 +0530 Subject: [PATCH 086/335] HDFS-15535. RBF: Fix Namespace path to snapshot path resolution for snapshot API. Contributed by Ayush Saxena. --- .../federation/router/RouterRpcClient.java | 50 ++++++++++++++++--- .../federation/router/RouterSnapshot.java | 18 ++++--- ...MultipleDestinationMountTableResolver.java | 33 ++++++++++++ 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index dae4b9356436c..fb068bfc3313a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -849,6 +849,45 @@ public T invokeSequential( final List locations, final RemoteMethod remoteMethod, Class expectedResultClass, Object expectedResultValue) throws IOException { + return (T) invokeSequential(remoteMethod, locations, expectedResultClass, + expectedResultValue).getResult(); + } + + /** + * Invokes sequential proxy calls to different locations. Continues to invoke + * calls until the success condition is met, or until all locations have been + * attempted. + * + * The success condition may be specified by: + *

    + *
  • An expected result class + *
  • An expected result value + *
+ * + * If no expected result class/values are specified, the success condition is + * a call that does not throw a remote exception. + * + * This returns RemoteResult, which contains the invoked location as well + * as the result. + * + * @param The type of the remote location. + * @param The type of the remote method return. + * @param remoteMethod The remote method and parameters to invoke. + * @param locations List of locations/nameservices to call concurrently. + * @param expectedResultClass In order to be considered a positive result, the + * return type must be of this class. + * @param expectedResultValue In order to be considered a positive result, the + * return value must equal the value of this object. + * @return The result of the first successful call, or if no calls are + * successful, the result of the first RPC call executed, along with + * the invoked location in form of RemoteResult. + * @throws IOException if the success condition is not met, return the first + * remote exception generated. + */ + public RemoteResult invokeSequential( + final RemoteMethod remoteMethod, final List locations, + Class expectedResultClass, Object expectedResultValue) + throws IOException { final UserGroupInformation ugi = RouterRpcServer.getRemoteUser(); final Method m = remoteMethod.getMethod(); @@ -867,9 +906,9 @@ public T invokeSequential( if (isExpectedClass(expectedResultClass, result) && isExpectedValue(expectedResultValue, result)) { // Valid result, stop here - @SuppressWarnings("unchecked") - T ret = (T)result; - return ret; + @SuppressWarnings("unchecked") R location = (R) loc; + @SuppressWarnings("unchecked") T ret = (T) result; + return new RemoteResult<>(location, ret); } if (firstResult == null) { firstResult = result; @@ -907,9 +946,8 @@ public T invokeSequential( throw thrownExceptions.get(0); } // Return the first result, whether it is the value or not - @SuppressWarnings("unchecked") - T ret = (T)firstResult; - return ret; + @SuppressWarnings("unchecked") T ret = (T) firstResult; + return new RemoteResult<>(locations.get(0), ret); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java index 3f4f4cb46bd82..9a47f2a012a0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterSnapshot.java @@ -104,10 +104,12 @@ public String createSnapshot(String snapshotRoot, String snapshotName) result = firstelement.getValue(); result = result.replaceFirst(loc.getDest(), loc.getSrc()); } else { - result = rpcClient.invokeSequential( - locations, method, String.class, null); - RemoteLocation loc = locations.get(0); - result = result.replaceFirst(loc.getDest(), loc.getSrc()); + RemoteResult response = + rpcClient.invokeSequential(method, locations, String.class, null); + RemoteLocation loc = response.getLocation(); + String invokedResult = response.getResult(); + result = invokedResult + .replaceFirst(loc.getDest(), loc.getSrc()); } return result; } @@ -180,9 +182,11 @@ public SnapshotStatus[] getSnapshotListing(String snapshotRoot) s.setParentFullPath(DFSUtil.string2Bytes(mountPath)); } } else { - response = rpcClient.invokeSequential( - locations, remoteMethod, SnapshotStatus[].class, null); - RemoteLocation loc = locations.get(0); + RemoteResult invokedResponse = rpcClient + .invokeSequential(remoteMethod, locations, SnapshotStatus[].class, + null); + RemoteLocation loc = invokedResponse.getLocation(); + response = invokedResponse.getResult(); for (SnapshotStatus s : response) { String mountPath = DFSUtil.bytes2String(s.getParentFullPath()). replaceFirst(loc.getDest(), loc.getSrc()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java index d00b93c43062c..181442d64708b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; @@ -492,6 +493,38 @@ public void testIsMultiDestDir() throws Exception { assertFalse(client.isMultiDestDirectory("/mount/dir")); } + /** + * Verifies the snapshot location returned after snapshot operations is in + * accordance to the mount path. + */ + @Test + public void testSnapshotPathResolution() throws Exception { + // Create a mount entry with non isPathAll order, so as to call + // invokeSequential. + Map destMap = new HashMap<>(); + destMap.put("ns0", "/tmp_ns0"); + destMap.put("ns1", "/tmp_ns1"); + nnFs0.mkdirs(new Path("/tmp_ns0")); + nnFs1.mkdirs(new Path("/tmp_ns1")); + MountTable addEntry = MountTable.newInstance("/mountSnap", destMap); + addEntry.setDestOrder(DestinationOrder.HASH); + assertTrue(addMountTable(addEntry)); + // Create the actual directory in the destination second in sequence of + // invokeSequential. + nnFs0.mkdirs(new Path("/tmp_ns0/snapDir")); + Path snapDir = new Path("/mountSnap/snapDir"); + Path snapshotPath = new Path("/mountSnap/snapDir/.snapshot/snap"); + routerFs.allowSnapshot(snapDir); + // Verify the snapshot path returned after createSnapshot is as per mount + // path. + Path snapshot = routerFs.createSnapshot(snapDir, "snap"); + assertEquals(snapshotPath, snapshot); + // Verify the snapshot path returned as part of snapshotListing is as per + // mount path. + SnapshotStatus[] snapshots = routerFs.getSnapshotListing(snapDir); + assertEquals(snapshotPath, snapshots[0].getFullPath()); + } + @Test public void testRenameMultipleDestDirectories() throws Exception { // Test renaming directories using rename API. From f734455e5d76c75b7d3a0b751023f5bd02ba38d2 Mon Sep 17 00:00:00 2001 From: Chen Liang Date: Thu, 20 Aug 2020 15:25:10 -0700 Subject: [PATCH 087/335] HDFS-15290. NPE in HttpServer during NameNode startup. Contributed by Simbarashe Dzinamarira. --- .../hdfs/server/namenode/ImageServlet.java | 17 ++++++- .../hdfs/server/namenode/NameNodeAdapter.java | 12 +++++ .../namenode/ha/TestStandbyCheckpoints.java | 45 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index a9c2a09ed48af..e045afbb64f43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -112,12 +112,25 @@ static void setRecentImageCheckTimePrecision(double ratio) { recentImageCheckTimePrecision = ratio; } + private FSImage getAndValidateFSImage(ServletContext context, + final HttpServletResponse response) + throws IOException { + final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); + if (nnImage == null) { + String errorMsg = "NameNode initialization not yet complete. " + + "FSImage has not been set in the NameNode."; + response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + throw new IOException(errorMsg); + } + return nnImage; + } + @Override public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException { try { final ServletContext context = getServletContext(); - final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); + final FSImage nnImage = getAndValidateFSImage(context, response); final GetImageParams parsedParams = new GetImageParams(request, response); final Configuration conf = (Configuration) context .getAttribute(JspHelper.CURRENT_CONF); @@ -524,7 +537,7 @@ protected void doPut(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException { try { ServletContext context = getServletContext(); - final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); + final FSImage nnImage = getAndValidateFSImage(context, response); final Configuration conf = (Configuration) getServletContext() .getAttribute(JspHelper.CURRENT_CONF); final PutImageParams parsedParams = new PutImageParams(request, response, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index 324f4fbe952f9..a584da1109f18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -56,6 +56,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.test.Whitebox; import org.mockito.Mockito; +import static org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer.FSIMAGE_ATTRIBUTE_KEY; /** * This is a utility class to expose NameNode functionality for unit tests. @@ -129,6 +130,17 @@ public static Server getRpcServer(NameNode namenode) { return ((NameNodeRpcServer)namenode.getRpcServer()).clientRpcServer; } + /** + * Sets the FSImage used in the NameNodeHttpServer and returns the old value. + */ + public static FSImage getAndSetFSImageInHttpServer(NameNode namenode, + FSImage fsImage) { + FSImage previous = (FSImage) namenode.httpServer.getHttpServer() + .getAttribute(FSIMAGE_ATTRIBUTE_KEY); + namenode.httpServer.setFSImage(fsImage); + return previous; + } + public static DelegationTokenSecretManager getDtSecretManager( final FSNamesystem ns) { return ns.getDelegationTokenSecretManager(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java index f96e8045d8607..1afd88d2235bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java @@ -21,6 +21,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import org.apache.hadoop.hdfs.LogVerificationAppender; +import org.apache.log4j.spi.LoggingEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -287,6 +289,49 @@ public void testStandbyAndObserverState() throws Exception { cluster.transitionToStandby(2); } + /** + * Tests that a null FSImage is handled gracefully by the ImageServlet. + * If putImage is called while a NameNode is still starting up, the FSImage + * may not have been initialized yet. See HDFS-15290. + */ + @Test(timeout = 30000) + public void testCheckpointBeforeNameNodeInitializationIsComplete() + throws Exception { + final LogVerificationAppender appender = new LogVerificationAppender(); + final org.apache.log4j.Logger logger = org.apache.log4j.Logger + .getRootLogger(); + logger.addAppender(appender); + + // Transition 2 to observer + cluster.transitionToObserver(2); + doEdits(0, 10); + // After a rollEditLog, Standby(nn1)'s next checkpoint would be + // ahead of observer(nn2). + nns[0].getRpcServer().rollEditLog(); + + NameNode nn2 = nns[2]; + FSImage nnFSImage = NameNodeAdapter.getAndSetFSImageInHttpServer(nn2, null); + + // After standby creating a checkpoint, it will try to push the image to + // active and all observer, updating it's own txid to the most recent. + HATestUtil.waitForCheckpoint(cluster, 1, ImmutableList.of(12)); + HATestUtil.waitForCheckpoint(cluster, 0, ImmutableList.of(12)); + + NameNodeAdapter.getAndSetFSImageInHttpServer(nn2, nnFSImage); + cluster.transitionToStandby(2); + logger.removeAppender(appender); + + for (LoggingEvent event : appender.getLog()) { + String message = event.getRenderedMessage(); + if (message.contains("PutImage failed") && + message.contains("FSImage has not been set in the NameNode.")) { + //Logs have the expected exception. + return; + } + } + fail("Expected exception not present in logs."); + } + /** * Test for the case when the SBN is configured to checkpoint based * on a time period, but no transactions are happening on the From 872c2909bdc636ec2c7da3f94b9e07348e3a6f0f Mon Sep 17 00:00:00 2001 From: swamirishi <47532440+swamirishi@users.noreply.github.com> Date: Sat, 22 Aug 2020 23:18:21 +0530 Subject: [PATCH 088/335] HADOOP-17122: Preserving Directory Attributes in DistCp with Atomic Copy (#2133) Contributed by Swaminathan Balachandran --- .../hadoop/tools/mapred/CopyCommitter.java | 6 +- .../tools/mapred/TestCopyCommitter.java | 69 +++++++++++++++---- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 139bd08fd7abc..e346d0b938c93 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -318,8 +318,10 @@ private void preserveFileAttributesForDirectories(Configuration conf) SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(sourceListing)); long totalLen = clusterFS.getFileStatus(sourceListing).getLen(); - - Path targetRoot = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); + // For Atomic Copy the Final & Work Path are different & atomic copy has + // already moved it to final path. + Path targetRoot = + new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); long preservedEntries = 0; try { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java index 11118c1f72400..685f030e15ea0 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java @@ -53,6 +53,8 @@ import java.util.*; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; +import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH; +import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_TARGET_WORK_PATH; import static org.apache.hadoop.tools.util.TestDistCpUtils.*; public class TestCopyCommitter { @@ -160,10 +162,10 @@ public void testPreserveStatus() throws IOException { context.setTargetPathExists(false); CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS); - Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong())); + Path listingFile = new Path("/tmp1/" + rand.nextLong()); listing.buildListing(listingFile, context); - conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, targetBase); committer.commitJob(jobContext); checkDirectoryPermissions(fs, targetBase, sourcePerm); @@ -179,6 +181,45 @@ public void testPreserveStatus() throws IOException { } + @Test + public void testPreserveStatusWithAtomicCommit() throws IOException { + TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config); + JobContext jobContext = new JobContextImpl( + taskAttemptContext.getConfiguration(), + taskAttemptContext.getTaskAttemptID().getJobID()); + Configuration conf = jobContext.getConfiguration(); + String sourceBase; + String workBase; + String targetBase; + FileSystem fs = null; + try { + OutputCommitter committer = new CopyCommitter(null, taskAttemptContext); + fs = FileSystem.get(conf); + FsPermission sourcePerm = new FsPermission((short) 511); + FsPermission initialPerm = new FsPermission((short) 448); + sourceBase = TestDistCpUtils.createTestSetup(fs, sourcePerm); + workBase = TestDistCpUtils.createTestSetup(fs, initialPerm); + targetBase = "/tmp1/" + rand.nextLong(); + final DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(new Path(sourceBase)), new Path("/out")) + .preserve(FileAttribute.PERMISSION).build(); + options.appendToConf(conf); + final DistCpContext context = new DistCpContext(options); + context.setTargetPathExists(false); + CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS); + Path listingFile = new Path("/tmp1/" + rand.nextLong()); + listing.buildListing(listingFile, context); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_WORK_PATH, workBase); + conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true); + committer.commitJob(jobContext); + checkDirectoryPermissions(fs, targetBase, sourcePerm); + } finally { + TestDistCpUtils.delete(fs, "/tmp1"); + conf.unset(DistCpConstants.CONF_LABEL_PRESERVE_STATUS); + } + } + @Test public void testDeleteMissing() throws IOException { TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config); @@ -207,8 +248,8 @@ public void testDeleteMissing() throws IOException { Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong())); listing.buildListing(listingFile, context); - conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); - conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_WORK_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, targetBase); committer.commitJob(jobContext); verifyFoldersAreInSync(fs, targetBase, sourceBase); @@ -256,8 +297,8 @@ public void testPreserveTimeWithDeleteMiss() throws IOException { Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong())); listing.buildListing(listingFile, context); - conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); - conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_WORK_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, targetBase); Path sourceListing = new Path( conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH)); @@ -320,8 +361,8 @@ public void testDeleteMissingFlatInterleavedFiles() throws IOException { Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong())); listing.buildListing(listingFile, context); - conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); - conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_WORK_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, targetBase); committer.commitJob(jobContext); verifyFoldersAreInSync(fs, targetBase, sourceBase); @@ -353,8 +394,8 @@ public void testAtomicCommitMissingFinal() throws IOException { fs = FileSystem.get(conf); fs.mkdirs(new Path(workPath)); - conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath); - conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath); + conf.set(CONF_LABEL_TARGET_WORK_PATH, workPath); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, finalPath); conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true); assertPathExists(fs, "Work path", new Path(workPath)); @@ -391,8 +432,8 @@ public void testAtomicCommitExistingFinal() throws IOException { fs.mkdirs(new Path(workPath)); fs.mkdirs(new Path(finalPath)); - conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath); - conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, finalPath); + conf.set(CONF_LABEL_TARGET_WORK_PATH, workPath); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, finalPath); conf.setBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, true); assertPathExists(fs, "Work path", new Path(workPath)); @@ -463,8 +504,8 @@ private void testCommitWithChecksumMismatch(boolean skipCrc) + String.valueOf(rand.nextLong())); listing.buildListing(listingFile, context); - conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); - conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_WORK_PATH, targetBase); + conf.set(CONF_LABEL_TARGET_FINAL_PATH, targetBase); OutputCommitter committer = new CopyCommitter( null, taskAttemptContext); From 83fd15b412c4e614defa90f34abb034fd2049231 Mon Sep 17 00:00:00 2001 From: hemanthboyina Date: Sun, 23 Aug 2020 16:10:47 +0530 Subject: [PATCH 089/335] HDFS-14504. Rename with Snapshots does not honor quota limit. --- .../hdfs/server/namenode/FSDirectory.java | 3 +- .../namenode/snapshot/TestSnapshotRename.java | 148 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 812b35f67d744..ae1afb434b42c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1210,7 +1210,8 @@ static void verifyQuota(INodesInPath iip, int pos, QuotaCounts deltas, // check existing components in the path for(int i = (pos > iip.length() ? iip.length(): pos) - 1; i >= 0; i--) { - if (commonAncestor == iip.getINode(i)) { + if (commonAncestor == iip.getINode(i) + && !commonAncestor.isInLatestSnapshot(iip.getLatestSnapshotId())) { // Stop checking for quota when common ancestor is reached return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java index 1d46e4ee92985..818f56ae3854d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -41,6 +42,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -280,4 +282,150 @@ public void testRenameSnapshotCommandWithIllegalArguments() throws Exception { System.setErr(oldErr); } } + + // Test rename of a snapshotted by setting quota in same directory. + @Test + public void testQuotaAndRenameWithSnapshot() throws Exception { + String dirr = "/dir2"; + Path dir2 = new Path(dirr); + Path fil1 = new Path(dir2, "file1"); + hdfs.mkdirs(dir2); + hdfs.setQuota(dir2, 3, 0); + hdfs.allowSnapshot(dir2); + hdfs.create(fil1); + hdfs.createSnapshot(dir2, "snap1"); + Path file2 = new Path(dir2, "file2"); + hdfs.rename(fil1, file2); + hdfs.getFileStatus(dir2); + Path filex = new Path(dir2, "filex"); + // create a file after exceeding namespace quota + LambdaTestUtils.intercept(NSQuotaExceededException.class, + "The NameSpace quota (directories and files) of " + + "directory /dir2 is exceeded", + () -> hdfs.create(filex)); + hdfs.createSnapshot(dir2, "snap2"); + // rename a file after exceeding namespace quota + Path file3 = new Path(dir2, "file3"); + LambdaTestUtils + .intercept(NSQuotaExceededException.class, + "The NameSpace quota (directories and files) of" + + " directory /dir2 is exceeded", + () -> hdfs.rename(file2, file3)); + } + + // Test Rename across directories within snapshot with quota set on source + // directory. + @Test + public void testRenameAcrossDirWithinSnapshot() throws Exception { + // snapshottable directory + String dirr = "/dir"; + Path rootDir = new Path(dirr); + hdfs.mkdirs(rootDir); + hdfs.allowSnapshot(rootDir); + + // set quota for source directory under snapshottable root directory + Path dir2 = new Path(rootDir, "dir2"); + Path fil1 = new Path(dir2, "file1"); + hdfs.mkdirs(dir2); + hdfs.setQuota(dir2, 3, 0); + hdfs.create(fil1); + Path file2 = new Path(dir2, "file2"); + hdfs.rename(fil1, file2); + Path fil3 = new Path(dir2, "file3"); + hdfs.create(fil3); + + // destination directory under snapshottable root directory + Path dir1 = new Path(rootDir, "dir1"); + Path dir1fil1 = new Path(dir1, "file1"); + hdfs.mkdirs(dir1); + hdfs.create(dir1fil1); + Path dir1fil2 = new Path(dir1, "file2"); + hdfs.rename(dir1fil1, dir1fil2); + + hdfs.createSnapshot(rootDir, "snap1"); + Path filex = new Path(dir2, "filex"); + // create a file after exceeding namespace quota + LambdaTestUtils.intercept(NSQuotaExceededException.class, + "The NameSpace quota (directories and files) of " + + "directory /dir/dir2 is exceeded", + () -> hdfs.create(filex)); + + // Rename across directories within snapshot with quota set on source + // directory + assertTrue(hdfs.rename(fil3, dir1)); + } + + // Test rename within the same directory within a snapshottable root with + // quota set. + @Test + public void testRenameInSameDirWithSnapshotableRoot() throws Exception { + + // snapshottable root directory + String rootStr = "/dir"; + Path rootDir = new Path(rootStr); + hdfs.mkdirs(rootDir); + hdfs.setQuota(rootDir, 3, 0); + hdfs.allowSnapshot(rootDir); + + // rename to be performed directory + String dirr = "dir2"; + Path dir2 = new Path(rootDir, dirr); + Path fil1 = new Path(dir2, "file1"); + hdfs.mkdirs(dir2); + hdfs.create(fil1); + hdfs.createSnapshot(rootDir, "snap1"); + Path file2 = new Path(dir2, "file2"); + // rename a file after exceeding namespace quota + LambdaTestUtils + .intercept(NSQuotaExceededException.class, + "The NameSpace quota (directories and files) of" + + " directory /dir is exceeded", + () -> hdfs.rename(fil1, file2)); + + } + + // Test rename from a directory under snapshottable root to a directory with + // quota set to a directory not under under any snapshottable root. + @Test + public void testRenameAcrossDirWithSnapshotableSrc() throws Exception { + // snapshottable directory + String dirr = "/dir"; + Path rootDir = new Path(dirr); + hdfs.mkdirs(rootDir); + hdfs.allowSnapshot(rootDir); + + // set quota for source directory + Path dir2 = new Path(rootDir, "dir2"); + Path fil1 = new Path(dir2, "file1"); + hdfs.mkdirs(dir2); + hdfs.setQuota(dir2, 3, 0); + hdfs.create(fil1); + Path file2 = new Path(dir2, "file2"); + hdfs.rename(fil1, file2); + Path fil3 = new Path(dir2, "file3"); + hdfs.create(fil3); + + hdfs.createSnapshot(rootDir, "snap1"); + + // destination directory not under any snapshot + String dirr1 = "/dir1"; + Path dir1 = new Path(dirr1); + Path dir1fil1 = new Path(dir1, "file1"); + hdfs.mkdirs(dir1); + hdfs.create(dir1fil1); + Path dir1fil2 = new Path(dir1, "file2"); + hdfs.rename(dir1fil1, dir1fil2); + + Path filex = new Path(dir2, "filex"); + // create a file after exceeding namespace quota on source + LambdaTestUtils.intercept(NSQuotaExceededException.class, + "The NameSpace quota (directories and files) of " + + "directory /dir/dir2 is exceeded", + () -> hdfs.create(filex)); + + // Rename across directories source dir under snapshot with quota set and + // destination directory not under any snapshot + assertTrue(hdfs.rename(fil3, dir1)); + } + } From 5c927eb550f3d064d13f653140b58725a3724e12 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 22 Jul 2020 15:40:37 +0530 Subject: [PATCH 090/335] YARN-10360. Support Multi Node Placement in SingleConstraintAppPlacementAllocator Reviewed by Sunil G. --- .../TestDSWithMultipleNodeManager.java | 55 ++++++++++++++++++- .../TestDistributedShell.java | 13 +++-- .../placement/AppPlacementAllocator.java | 40 +++++++++++++- .../LocalityAppPlacementAllocator.java | 39 ------------- ...SingleConstraintAppPlacementAllocator.java | 18 ------ 5 files changed, 100 insertions(+), 65 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java index f3571a6b81aa6..2aead20a1c373 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.Set; import java.util.Iterator; @@ -29,32 +31,83 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Test for Distributed Shell With Multiple Node Managers. + * Parameter 0 tests with Single Node Placement and + * parameter 1 tests with Multiple Node Placement. + */ +@RunWith(value = Parameterized.class) public class TestDSWithMultipleNodeManager { private static final Logger LOG = LoggerFactory.getLogger(TestDSWithMultipleNodeManager.class); static final int NUM_NMS = 2; TestDistributedShell distShellTest; + private final Boolean multiNodePlacementEnabled; + private static final String POLICY_CLASS_NAME = + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement." + + "ResourceUsageMultiNodeLookupPolicy"; + + + @Parameterized.Parameters + public static Collection getParams() { + return Arrays.asList(false, true); + } + + public TestDSWithMultipleNodeManager(Boolean multiNodePlacementEnabled) { + this.multiNodePlacementEnabled = multiNodePlacementEnabled; + } + + private YarnConfiguration getConfiguration( + boolean multiNodePlacementConfigs) { + YarnConfiguration conf = new YarnConfiguration(); + if (multiNodePlacementConfigs) { + conf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getName()); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.set(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICIES, + "resource-based"); + conf.set(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME, + "resource-based"); + String policyName = + CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + + ".resource-based" + ".class"; + conf.set(policyName, POLICY_CLASS_NAME); + conf.setBoolean( + CapacitySchedulerConfiguration.MULTI_NODE_PLACEMENT_ENABLED, true); + } + return conf; + } @Before public void setup() throws Exception { distShellTest = new TestDistributedShell(); - distShellTest.setupInternal(NUM_NMS); + distShellTest.setupInternal(NUM_NMS, + getConfiguration(multiNodePlacementEnabled)); } @After diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 0ecb841fe7401..41ba8dfa36b3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -141,18 +141,21 @@ public class TestDistributedShell { @Before public void setup() throws Exception { - setupInternal(NUM_NMS, timelineVersionWatcher.getTimelineVersion()); + setupInternal(NUM_NMS, timelineVersionWatcher.getTimelineVersion(), + new YarnConfiguration()); } - protected void setupInternal(int numNodeManager) throws Exception { - setupInternal(numNodeManager, DEFAULT_TIMELINE_VERSION); + protected void setupInternal(int numNodeManager, + YarnConfiguration yarnConfig) throws Exception { + setupInternal(numNodeManager, DEFAULT_TIMELINE_VERSION, yarnConfig); } - private void setupInternal(int numNodeManager, float timelineVersion) + private void setupInternal(int numNodeManager, float timelineVersion, + YarnConfiguration yarnConfig) throws Exception { LOG.info("Starting up YARN cluster"); - conf = new YarnConfiguration(); + this.conf = yarnConfig; conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, MIN_ALLOCATION_MB); // reduce the teardown waiting time diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java index d71b9a0f7774d..b1b340269d87f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement; +import org.apache.commons.collections.IteratorUtils; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.DiagnosticsCollector; @@ -26,9 +27,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ApplicationSchedulingConfig; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Iterator; @@ -59,14 +63,35 @@ public abstract class AppPlacementAllocator { protected SchedulerRequestKey schedulerRequestKey; protected RMContext rmContext; private AtomicInteger placementAttempt = new AtomicInteger(0); + private MultiNodeSortingManager multiNodeSortingManager = null; + private String multiNodeSortPolicyName; + + private static final Logger LOG = + LoggerFactory.getLogger(AppPlacementAllocator.class); /** * Get iterator of preferred node depends on requirement and/or availability. * @param candidateNodeSet input CandidateNodeSet * @return iterator of preferred node */ - public abstract Iterator getPreferredNodeIterator( - CandidateNodeSet candidateNodeSet); + public Iterator getPreferredNodeIterator( + CandidateNodeSet candidateNodeSet) { + // Now only handle the case that single node in the candidateNodeSet + // TODO, Add support to multi-hosts inside candidateNodeSet which is passed + // in. + + N singleNode = CandidateNodeSetUtils.getSingleNode(candidateNodeSet); + if (singleNode != null) { + return IteratorUtils.singletonIterator(singleNode); + } + + // singleNode will be null if Multi-node placement lookup is enabled, and + // hence could consider sorting policies. + return multiNodeSortingManager.getMultiNodeSortIterator( + candidateNodeSet.getAllNodes().values(), + candidateNodeSet.getPartition(), + multiNodeSortPolicyName); + } /** * Replace existing pending asks by the new requests @@ -200,6 +225,17 @@ public void initialize(AppSchedulingInfo appSchedulingInfo, this.appSchedulingInfo = appSchedulingInfo; this.rmContext = rmContext; this.schedulerRequestKey = schedulerRequestKey; + multiNodeSortPolicyName = appSchedulingInfo + .getApplicationSchedulingEnvs().get( + ApplicationSchedulingConfig.ENV_MULTI_NODE_SORTING_POLICY_CLASS); + multiNodeSortingManager = (MultiNodeSortingManager) rmContext + .getMultiNodeSortingManager(); + if (LOG.isDebugEnabled()) { + LOG.debug( + "nodeLookupPolicy used for " + appSchedulingInfo.getApplicationId() + + " is " + ((multiNodeSortPolicyName != null) + ? multiNodeSortPolicyName : "")); + } } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java index a91e87246f734..19ba719c30bf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement; -import org.apache.commons.collections.IteratorUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.DiagnosticsCollector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,14 +31,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ApplicationSchedulingConfig; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -60,8 +57,6 @@ public class LocalityAppPlacementAllocator new ConcurrentHashMap<>(); private volatile String primaryRequestedPartition = RMNodeLabelsManager.NO_LABEL; - private MultiNodeSortingManager multiNodeSortingManager = null; - private String multiNodeSortPolicyName; private final ReentrantReadWriteLock.ReadLock readLock; private final ReentrantReadWriteLock.WriteLock writeLock; @@ -77,40 +72,6 @@ public LocalityAppPlacementAllocator() { public void initialize(AppSchedulingInfo appSchedulingInfo, SchedulerRequestKey schedulerRequestKey, RMContext rmContext) { super.initialize(appSchedulingInfo, schedulerRequestKey, rmContext); - multiNodeSortPolicyName = appSchedulingInfo - .getApplicationSchedulingEnvs().get( - ApplicationSchedulingConfig.ENV_MULTI_NODE_SORTING_POLICY_CLASS); - multiNodeSortingManager = (MultiNodeSortingManager) rmContext - .getMultiNodeSortingManager(); - if (LOG.isDebugEnabled()) { - LOG.debug( - "nodeLookupPolicy used for " + appSchedulingInfo - .getApplicationId() - + " is " + ((multiNodeSortPolicyName != null) ? - multiNodeSortPolicyName : - "")); - } - } - - @Override - @SuppressWarnings("unchecked") - public Iterator getPreferredNodeIterator( - CandidateNodeSet candidateNodeSet) { - // Now only handle the case that single node in the candidateNodeSet - // TODO, Add support to multi-hosts inside candidateNodeSet which is passed - // in. - - N singleNode = CandidateNodeSetUtils.getSingleNode(candidateNodeSet); - if (singleNode != null) { - return IteratorUtils.singletonIterator(singleNode); - } - - // singleNode will be null if Multi-node placement lookup is enabled, and - // hence could consider sorting policies. - return multiNodeSortingManager.getMultiNodeSortIterator( - candidateNodeSet.getAllNodes().values(), - candidateNodeSet.getPartition(), - multiNodeSortPolicyName); } private boolean hasRequestLabelChanged(ResourceRequest requestOne, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index 9898051bd9bc2..fa468beb9c3d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.collections.IteratorUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.DiagnosticsCollector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +45,6 @@ import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -78,22 +76,6 @@ public SingleConstraintAppPlacementAllocator() { writeLock = lock.writeLock(); } - @Override - @SuppressWarnings("unchecked") - public Iterator getPreferredNodeIterator( - CandidateNodeSet candidateNodeSet) { - // Now only handle the case that single node in the candidateNodeSet - // TODO, Add support to multi-hosts inside candidateNodeSet which is passed - // in. - - N singleNode = CandidateNodeSetUtils.getSingleNode(candidateNodeSet); - if (null != singleNode) { - return IteratorUtils.singletonIterator(singleNode); - } - - return IteratorUtils.emptyIterator(); - } - @Override public PendingAskUpdateResult updatePendingAsk( Collection requests, From 17cd8a1b1627f4d87ddc5dcc2ec7c738ffdc576b Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 24 Aug 2020 11:24:31 +0100 Subject: [PATCH 091/335] HADOOP-17209. Erasure Coding: Native library memory leak. Contriubted by Sean Chow --- .../native/src/org/apache/hadoop/io/erasurecode/jni_common.c | 4 +++- .../src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c index 9cca6dd754b09..816536b637d39 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c @@ -92,6 +92,7 @@ void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets, destInputs[i] = NULL; } } + (*env)->ReleaseIntArrayElements(env, inputOffsets, tmpInputOffsets, 0); } void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets, @@ -112,4 +113,5 @@ void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets, byteBuffer)); destOutputs[i] += tmpOutputOffsets[i]; } -} \ No newline at end of file + (*env)->ReleaseIntArrayElements(env, outputOffsets, tmpOutputOffsets, 0); +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c index 52d255afd58d8..72314d2ad545a 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c @@ -66,6 +66,7 @@ jintArray outputOffsets) { decode(&rsDecoder->decoder, rsDecoder->inputs, tmpErasedIndexes, numErased, rsDecoder->outputs, chunkSize); + (*env)->ReleaseIntArrayElements(env, erasedIndexes, tmpErasedIndexes, 0); } JNIEXPORT void JNICALL From 960fb0aa4f5a6f8b35ee210248f69ecfe25ddb15 Mon Sep 17 00:00:00 2001 From: Joey <540260711@qq.com> Date: Mon, 24 Aug 2020 21:01:48 +0800 Subject: [PATCH 092/335] HADOOP-16925. MetricsConfig incorrectly loads the configuration whose value is String list in the properties file (#1896) Contributed by Jiayi Liu --- .../hadoop/metrics2/impl/MetricsConfig.java | 2 ++ .../hadoop/metrics2/impl/TestMetricsConfig.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java index 976f16bedd81b..a1f4d2391f6ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java @@ -37,6 +37,7 @@ import org.apache.commons.configuration2.Configuration; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.SubsetConfiguration; +import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.configuration2.io.FileHandler; import org.apache.hadoop.metrics2.MetricsFilter; @@ -111,6 +112,7 @@ static MetricsConfig loadFirst(String prefix, String... fileNames) { for (String fname : fileNames) { try { PropertiesConfiguration pcf = new PropertiesConfiguration(); + pcf.setListDelimiterHandler(new DefaultListDelimiterHandler(',')); FileHandler fh = new FileHandler(pcf); fh.setFileName(fname); fh.load(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java index b53be4d73599a..2ca1c8ad2cc35 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsConfig.java @@ -133,6 +133,22 @@ private void testInstances(MetricsConfig c) throws Exception { assertEq(expected, mc2); } + /** + * Test the config value separated by delimiter + */ + @Test public void testDelimiterConf() { + String filename = getTestFilename("test-metrics2-delimiter"); + new ConfigBuilder().add("p1.foo", "p1foo1,p1foo2,p1foo3").save(filename); + + MetricsConfig mc = MetricsConfig.create("p1", filename); + Configuration expected = new ConfigBuilder() + .add("foo", "p1foo1") + .add("foo", "p1foo2") + .add("foo", "p1foo3") + .config; + assertEq(expected, mc); + } + /** * Return a test filename in the class path * @param basename From a7830423c578d2909112f90aedf3d9aefeabc9c2 Mon Sep 17 00:00:00 2001 From: He Xiaoqiao Date: Mon, 24 Aug 2020 21:13:47 +0800 Subject: [PATCH 093/335] HDFS-15448. Remove duplicate BlockPoolManager starting when run DataNode. Contriubted by jianghua zhu. --- .../apache/hadoop/hdfs/server/datanode/DataNode.java | 6 +++++- .../hadoop/hdfs/server/datanode/TestDataNodeExit.java | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 7f740c39d8cf2..36ec1f4aceca9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -2710,7 +2710,11 @@ void closeBlock(ExtendedBlock block, String delHint, String storageUuid, * If this thread is specifically interrupted, it will stop waiting. */ public void runDatanodeDaemon() throws IOException { - blockPoolManager.startAll(); + + // Verify that blockPoolManager has been started. + if (!isDatanodeUp()) { + throw new IOException("Failed to instantiate DataNode."); + } // start dataXceiveServer dataXceiverServer.start(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java index 918ad83bbd437..8c2fe37a58737 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java @@ -82,6 +82,17 @@ private void stopBPServiceThreads(int numStopThreads, DataNode dn) dn.getBpOsCount()); } + @Test + public void testBPServiceState() { + List dataNodes = cluster.getDataNodes(); + for (DataNode dataNode : dataNodes) { + List bposList = dataNode.getAllBpOs(); + for (BPOfferService bpOfferService : bposList) { + assertTrue(bpOfferService.isAlive()); + } + } + } + /** * Test BPService Thread Exit */ From 15a0fed637129be04bbbb9300eb363341f8f5365 Mon Sep 17 00:00:00 2001 From: Brahma Reddy Battula Date: Mon, 24 Aug 2020 19:03:22 +0530 Subject: [PATCH 094/335] HADOOP-17220. Upgrade slf4j to 1.7.30 ( To Address: CVE-2018-8088). Contributed by Brahma Reddy Battula. --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 373450c822bda..12ee1399bb31e 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -79,7 +79,7 @@ 4.4.10 - 1.7.25 + 1.7.30 1.2.17 From 64f36b9543c011ce2f1f7d1e10da0eab88a0759d Mon Sep 17 00:00:00 2001 From: bilaharith <52483117+bilaharith@users.noreply.github.com> Date: Tue, 25 Aug 2020 00:30:55 +0530 Subject: [PATCH 095/335] HADOOP-16915. ABFS: Ignoring the test ITestAzureBlobFileSystemRandomRead.testRandomReadPerformance - Contributed by Bilahari T H --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java index e5f64b5f2c0a9..f58276312831a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java @@ -23,6 +23,7 @@ import java.util.concurrent.Callable; import org.junit.Assume; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -412,6 +413,7 @@ public void testSequentialReadAfterReverseSeekPerformance() } @Test + @Ignore("HADOOP-16915") public void testRandomReadPerformance() throws Exception { Assume.assumeFalse("This test does not support namespace enabled account", this.getFileSystem().getIsNamespaceEnabled()); From a932796d0cad3d84df0003782e4247cbc2dcca93 Mon Sep 17 00:00:00 2001 From: sguggilam Date: Mon, 24 Aug 2020 23:39:57 -0700 Subject: [PATCH 096/335] HADOOP-17159 Ability for forceful relogin in UserGroupInformation class (#2197) Contributed by Sandeep Guggilam. Signed-off-by: Mingliang Liu Signed-off-by: Steve Loughran --- .../hadoop/security/UserGroupInformation.java | 35 ++++++++++++++---- .../security/TestUGILoginFromKeytab.java | 36 +++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 91b64ade598e5..d1ab4365a045e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1232,7 +1232,26 @@ public void reloginFromKeytab() throws IOException { reloginFromKeytab(false); } - private void reloginFromKeytab(boolean checkTGT) throws IOException { + /** + * Force re-Login a user in from a keytab file. Loads a user identity from a + * keytab file and logs them in. They become the currently logged-in user. + * This method assumes that {@link #loginUserFromKeytab(String, String)} had + * happened already. The Subject field of this UserGroupInformation object is + * updated to have the new credentials. + * + * @param ignoreTimeElapsed Force re-login irrespective of the time of last + * login + * @throws IOException + * @throws KerberosAuthException on a failure + */ + @InterfaceAudience.Public + @InterfaceStability.Evolving + public void reloginFromKeytab(boolean ignoreTimeElapsed) throws IOException { + reloginFromKeytab(false, ignoreTimeElapsed); + } + + private void reloginFromKeytab(boolean checkTGT, boolean ignoreTimeElapsed) + throws IOException { if (!shouldRelogin() || !isFromKeytab()) { return; } @@ -1247,7 +1266,7 @@ private void reloginFromKeytab(boolean checkTGT) throws IOException { return; } } - relogin(login); + relogin(login, ignoreTimeElapsed); } /** @@ -1268,25 +1287,27 @@ public void reloginFromTicketCache() throws IOException { if (login == null) { throw new KerberosAuthException(MUST_FIRST_LOGIN); } - relogin(login); + relogin(login, false); } - private void relogin(HadoopLoginContext login) throws IOException { + private void relogin(HadoopLoginContext login, boolean ignoreTimeElapsed) + throws IOException { // ensure the relogin is atomic to avoid leaving credentials in an // inconsistent state. prevents other ugi instances, SASL, and SPNEGO // from accessing or altering credentials during the relogin. synchronized(login.getSubjectLock()) { // another racing thread may have beat us to the relogin. if (login == getLogin()) { - unprotectedRelogin(login); + unprotectedRelogin(login, ignoreTimeElapsed); } } } - private void unprotectedRelogin(HadoopLoginContext login) throws IOException { + private void unprotectedRelogin(HadoopLoginContext login, + boolean ignoreTimeElapsed) throws IOException { assert Thread.holdsLock(login.getSubjectLock()); long now = Time.now(); - if (!hasSufficientTimeElapsed(now)) { + if (!hasSufficientTimeElapsed(now) && !ignoreTimeElapsed) { return; } // register most recent relogin attempt diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java index d233234c26c0c..bf4cf75ba8179 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java @@ -158,6 +158,42 @@ public void testUGIReLoginFromKeytab() throws Exception { Assert.assertNotSame(login1, login2); } + /** + * Force re-login from keytab using the MiniKDC and verify the UGI can + * successfully relogin from keytab as well. + */ + @Test + public void testUGIForceReLoginFromKeytab() throws Exception { + // Set this to false as we are testing force re-login anyways + UserGroupInformation.setShouldRenewImmediatelyForTests(false); + String principal = "foo"; + File keytab = new File(workDir, "foo.keytab"); + kdc.createPrincipal(keytab, principal); + + UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + Assert.assertTrue("UGI should be configured to login from keytab", + ugi.isFromKeytab()); + + // Verify relogin from keytab. + User user = getUser(ugi.getSubject()); + final long firstLogin = user.getLastLogin(); + final LoginContext login1 = user.getLogin(); + Assert.assertNotNull(login1); + + // Sleep for 2 secs to have a difference between first and second login + Thread.sleep(2000); + + // Force relogin from keytab + ugi.reloginFromKeytab(true); + final long secondLogin = user.getLastLogin(); + final LoginContext login2 = user.getLogin(); + Assert.assertTrue("User should have been able to relogin from keytab", + secondLogin > firstLogin); + Assert.assertNotNull(login2); + Assert.assertNotSame(login1, login2); + } + @Test public void testGetUGIFromKnownSubject() throws Exception { KerberosPrincipal principal = new KerberosPrincipal("user"); From c0492962355b862d055d0b88bf9e2874f5dd7041 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Tue, 25 Aug 2020 09:53:05 +0200 Subject: [PATCH 097/335] =?UTF-8?q?YARN-10106.=20Yarn=20logs=20CLI=20filte?= =?UTF-8?q?ring=20by=20application=20attempt.=20Contributed=20by=20Hud?= =?UTF-8?q?=C3=A1ky=20M=C3=A1rton=20Gyula?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hadoop/yarn/client/cli/LogsCLI.java | 40 +++- .../hadoop/yarn/client/cli/TestLogsCLI.java | 185 +++++++++++++++--- 2 files changed, 192 insertions(+), 33 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 4d67ce8178d6e..2ae890bdccc69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -65,6 +65,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -95,6 +96,8 @@ public class LogsCLI extends Configured implements Tool { private static final String CONTAINER_ID_OPTION = "containerId"; private static final String APPLICATION_ID_OPTION = "applicationId"; + private static final String APPLICATION_ATTEMPT_ID_OPTION = + "applicationAttemptId"; private static final String CLUSTER_ID_OPTION = "clusterId"; private static final String NODE_ADDRESS_OPTION = "nodeAddress"; private static final String APP_OWNER_OPTION = "appOwner"; @@ -160,6 +163,7 @@ private int runCommand(String[] args) throws Exception { } CommandLineParser parser = new GnuParser(); String appIdStr = null; + String appAttemptIdStr = null; String clusterIdStr = null; String containerIdStr = null; String nodeAddress = null; @@ -180,6 +184,8 @@ private int runCommand(String[] args) throws Exception { try { CommandLine commandLine = parser.parse(opts, args, false); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); + appAttemptIdStr = commandLine.getOptionValue( + APPLICATION_ATTEMPT_ID_OPTION); containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); @@ -240,9 +246,9 @@ private int runCommand(String[] args) throws Exception { return -1; } - if (appIdStr == null && containerIdStr == null) { - System.err.println("Both applicationId and containerId are missing, " - + " one of them must be specified."); + if (appIdStr == null && appAttemptIdStr == null && containerIdStr == null) { + System.err.println("None of applicationId, appAttemptId and containerId " + + "is available, one of them must be specified."); printHelpMessage(printOpts); return -1; } @@ -257,9 +263,32 @@ private int runCommand(String[] args) throws Exception { } } + ApplicationAttemptId appAttemptId = null; + if (appAttemptIdStr != null) { + try { + appAttemptId = ApplicationAttemptId.fromString(appAttemptIdStr); + if (appId == null) { + appId = appAttemptId.getApplicationId(); + } else if (!appId.equals(appAttemptId.getApplicationId())) { + System.err.println("The Application:" + appId + + " does not have the AppAttempt:" + appAttemptId); + return -1; + } + } catch (Exception e) { + System.err.println("Invalid AppAttemptId specified"); + return -1; + } + } + if (containerIdStr != null) { try { ContainerId containerId = ContainerId.fromString(containerIdStr); + if (appAttemptId != null && !appAttemptId.equals( + containerId.getApplicationAttemptId())) { + System.err.println("The AppAttempt:" + appAttemptId + + " does not have the container:" + containerId); + return -1; + } if (appId == null) { appId = containerId.getApplicationAttemptId().getApplicationId(); } else if (!containerId.getApplicationAttemptId().getApplicationId() @@ -344,7 +373,7 @@ private int runCommand(String[] args) throws Exception { } - ContainerLogsRequest request = new ContainerLogsRequest(appId, null, + ContainerLogsRequest request = new ContainerLogsRequest(appId, appAttemptId, Apps.isApplicationFinalState(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null); @@ -913,6 +942,9 @@ private Options createCommandOpts() { Option appIdOpt = new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)"); opts.addOption(appIdOpt); + opts.addOption(APPLICATION_ATTEMPT_ID_OPTION, true, "ApplicationAttemptId. " + + "Lists all logs belonging to the specified application attempt Id. " + + "If specified, the applicationId can be omitted"); opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId. " + "By default, it will print all available logs." + " Work with -log_files to get only specific logs. If specified, the" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 24256a0147142..39428919a0749 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -238,6 +238,22 @@ public void testUnknownApplicationId() throws Exception { "Unable to get ApplicationState")); } + @Test(timeout = 5000L) + public void testUnknownApplicationAttemptId() throws Exception { + YarnClient mockYarnClient = createMockYarnClientUnknownApp(); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(conf); + ApplicationId appId = ApplicationId.newInstance(0, 1); + + int exitCode = cli.run(new String[] {"-applicationAttemptId", + ApplicationAttemptId.newInstance(appId, 1).toString() }); + + // Error since no logs present for the app. + assertTrue(exitCode != 0); + assertTrue(sysErrStream.toString().contains( + "Unable to get ApplicationState.")); + } + @Test (timeout = 10000) public void testHelpMessage() throws Exception { YarnClient mockYarnClient = createMockYarnClient( @@ -372,12 +388,14 @@ public void testFetchFinishedApplictionLogs() throws Exception { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ApplicationId appId = ApplicationId.newInstance(0, 1); - ApplicationAttemptId appAttemptId = + ApplicationAttemptId appAttemptId1 = ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0); - ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); - ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2); - ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3); + ApplicationAttemptId appAttemptId2 = + ApplicationAttemptId.newInstance(appId, 2); + ContainerId containerId0 = ContainerId.newContainerId(appAttemptId1, 0); + ContainerId containerId1 = ContainerId.newContainerId(appAttemptId1, 1); + ContainerId containerId2 = ContainerId.newContainerId(appAttemptId1, 2); + ContainerId containerId3 = ContainerId.newContainerId(appAttemptId2, 3); final NodeId nodeId = NodeId.newInstance("localhost", 1234); // create local logs @@ -464,6 +482,44 @@ public ContainerReport getContainerReport(String containerIdStr) createEmptyLog("empty"))); sysOutStream.reset(); + // Check fetching data for application attempt with applicationId defined + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-applicationAttemptId", appAttemptId1.toString()}); + LOG.info(sysOutStream.toString()); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + assertTrue(sysOutStream.toString().contains( + createEmptyLog("empty"))); + sysOutStream.reset(); + + // Check fetching data for application attempt without application defined + exitCode = cli.run(new String[] { + "-applicationAttemptId", appAttemptId1.toString()}); + LOG.info(sysOutStream.toString()); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + assertTrue(sysOutStream.toString().contains( + createEmptyLog("empty"))); + sysOutStream.reset(); + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files_pattern", ".*"}); assertTrue(exitCode == 0); @@ -979,6 +1035,8 @@ public void testFetchRunningApplicationLogs() throws Exception { any(ContainerLogsRequest.class)); cli2.setConf(new YarnConfiguration()); ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); + System.out.println(containerId100.toString()); + System.out.println(appId.toString()); exitCode = cli2.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId100.toString(), "-nodeAddress", "NM:1234"}); assertTrue(exitCode == 0); @@ -1139,52 +1197,112 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception { } @Test (timeout = 5000) - public void testLogsCLIWithInvalidArgs() throws Exception { - String localDir = "target/SaveLogs"; - Path localPath = new Path(localDir); - FileSystem fs = FileSystem.get(conf); - ApplicationId appId = ApplicationId.newInstance(0, 1); - YarnClient mockYarnClient = - createMockYarnClient(YarnApplicationState.FINISHED, - UserGroupInformation.getCurrentUser().getShortUserName()); - LogsCLI cli = new LogsCLIForTest(mockYarnClient); - cli.setConf(conf); + public void testWithInvalidApplicationId() throws Exception { + LogsCLI cli = createCli(); // Specify an invalid applicationId - int exitCode = cli.run(new String[] {"-applicationId", - "123"}); + int exitCode = cli.run(new String[] {"-applicationId", "123"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( - "Invalid ApplicationId specified")); + "Invalid ApplicationId specified")); + } + + @Test (timeout = 5000) + public void testWithInvalidAppAttemptId() throws Exception { + LogsCLI cli = createCli(); + + // Specify an invalid appAttemptId + int exitCode = cli.run(new String[] {"-applicationAttemptId", "123"}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Invalid AppAttemptId specified")); sysErrStream.reset(); + } + + @Test (timeout = 5000) + public void testWithInvalidContainerId() throws Exception { + LogsCLI cli = createCli(); // Specify an invalid containerId - exitCode = cli.run(new String[] {"-containerId", - "123"}); + int exitCode = cli.run(new String[] {"-containerId", "123"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( - "Invalid ContainerId specified")); + "Invalid ContainerId specified")); + sysErrStream.reset(); + } + + @Test (timeout = 5000) + public void testWithNonMatchingEntityIds() throws Exception { + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + ApplicationId appId2 = ApplicationId.newInstance(0, 2); + ApplicationAttemptId appAttemptId1 = + ApplicationAttemptId.newInstance(appId1, 1); + ApplicationAttemptId appAttemptId2 = + ApplicationAttemptId.newInstance(appId2, 1); + ContainerId containerId0 = ContainerId.newContainerId(appAttemptId1, 0); + LogsCLI cli = createCli(); + + // Non-matching applicationId and applicationAttemptId + int exitCode = cli.run(new String[] {"-applicationId", appId2.toString(), + "-applicationAttemptId", appAttemptId1.toString()}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "The Application:" + appId2.toString() + + " does not have the AppAttempt:" + appAttemptId1.toString())); sysErrStream.reset(); + // Non-matching applicationId and containerId + exitCode = cli.run(new String[] {"-applicationId", appId2.toString(), + "-containerId", containerId0.toString()}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "The Application:" + appId2.toString() + + " does not have the container:" + containerId0.toString())); + sysErrStream.reset(); + + // Non-matching applicationAttemptId and containerId + exitCode = cli.run(new String[] {"-applicationAttemptId", + appAttemptId2.toString(), "-containerId", containerId0.toString()}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "The AppAttempt:" + appAttemptId2.toString() + + " does not have the container:" + containerId0.toString())); + sysErrStream.reset(); + } + + @Test (timeout = 5000) + public void testWithExclusiveArguments() throws Exception { + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + LogsCLI cli = createCli(); + // Specify show_container_log_info and show_application_log_info // at the same time - exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + int exitCode = cli.run(new String[] {"-applicationId", appId1.toString(), "-show_container_log_info", "-show_application_log_info"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains("Invalid options. " - + "Can only accept one of show_application_log_info/" - + "show_container_log_info.")); + + "Can only accept one of show_application_log_info/" + + "show_container_log_info.")); sysErrStream.reset(); // Specify log_files and log_files_pattern // at the same time - exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + exitCode = cli.run(new String[] {"-applicationId", appId1.toString(), "-log_files", "*", "-log_files_pattern", ".*"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains("Invalid options. " - + "Can only accept one of log_files/" - + "log_files_pattern.")); + + "Can only accept one of log_files/" + + "log_files_pattern.")); sysErrStream.reset(); + } + + @Test (timeout = 5000) + public void testWithFileInputForOptionOut() throws Exception { + String localDir = "target/SaveLogs"; + Path localPath = new Path(localDir); + FileSystem fs = FileSystem.get(conf); + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + LogsCLI cli = createCli(); // Specify a file name to the option -out try { @@ -1193,8 +1311,8 @@ public void testLogsCLIWithInvalidArgs() throws Exception { if (!fs.exists(tmpFilePath)) { fs.createNewFile(tmpFilePath); } - exitCode = cli.run(new String[] {"-applicationId", - appId.toString(), + int exitCode = cli.run(new String[] {"-applicationId", + appId1.toString(), "-out" , tmpFilePath.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( @@ -1708,6 +1826,15 @@ private static void uploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ug } } + private LogsCLI createCli() throws IOException, YarnException { + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(conf); + return cli; + } + private YarnClient createMockYarnClient(YarnApplicationState appState, String user) throws YarnException, IOException { return createMockYarnClient(appState, user, false, null, null); From cc641534dcb39d304ecbd0820963465b85677ce2 Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Tue, 25 Aug 2020 15:59:43 +0530 Subject: [PATCH 098/335] HADOOP-17074. S3A Listing to be fully asynchronous. (#2207) Contributed by Mukund Thakur. --- .../org/apache/hadoop/fs/s3a/Listing.java | 56 ++++++--- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 11 +- .../s3a/impl/ListingOperationCallbacks.java | 7 +- .../s3a/impl/TestPartialDeleteFailures.java | 6 +- .../scale/ITestS3ADirectoryPerformance.java | 107 +++++++++++++++++- 5 files changed, 164 insertions(+), 23 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index 34129e0bf1a74..16413a7620d0d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -55,7 +55,9 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import static org.apache.hadoop.fs.impl.FutureIOSupport.awaitFuture; import static org.apache.hadoop.fs.s3a.Constants.S3N_FOLDER_SUFFIX; import static org.apache.hadoop.fs.s3a.S3AUtils.ACCEPT_ALL; import static org.apache.hadoop.fs.s3a.S3AUtils.createFileStatus; @@ -739,6 +741,16 @@ class ObjectListingIterator implements RemoteIterator { */ private int maxKeys; + /** + * Future to store current batch listing result. + */ + private CompletableFuture s3ListResultFuture; + + /** + * Result of previous batch. + */ + private S3ListResult objectsPrev; + /** * Constructor -calls `listObjects()` on the request to populate the * initial set of results/fail if there was a problem talking to the bucket. @@ -752,8 +764,10 @@ class ObjectListingIterator implements RemoteIterator { S3ListRequest request) throws IOException { this.listPath = listPath; this.maxKeys = listingOperationCallbacks.getMaxKeys(); - this.objects = listingOperationCallbacks.listObjects(request); + this.s3ListResultFuture = listingOperationCallbacks + .listObjectsAsync(request); this.request = request; + this.objectsPrev = null; } /** @@ -764,7 +778,8 @@ class ObjectListingIterator implements RemoteIterator { */ @Override public boolean hasNext() throws IOException { - return firstListing || objects.isTruncated(); + return firstListing || + (objectsPrev != null && objectsPrev.isTruncated()); } /** @@ -780,29 +795,44 @@ public boolean hasNext() throws IOException { @Retries.RetryTranslated public S3ListResult next() throws IOException { if (firstListing) { - // on the first listing, don't request more data. - // Instead just clear the firstListing flag so that it future calls - // will request new data. + // clear the firstListing flag for future calls. firstListing = false; + // Calculating the result of last async list call. + objects = awaitFuture(s3ListResultFuture); + fetchNextBatchAsyncIfPresent(); } else { try { - if (!objects.isTruncated()) { + if (objectsPrev!= null && !objectsPrev.isTruncated()) { // nothing more to request: fail. throw new NoSuchElementException("No more results in listing of " - + listPath); + + listPath); } - // need to request a new set of objects. - LOG.debug("[{}], Requesting next {} objects under {}", - listingCount, maxKeys, listPath); - objects = listingOperationCallbacks - .continueListObjects(request, objects); + // Calculating the result of last async list call. + objects = awaitFuture(s3ListResultFuture); + // Requesting next batch of results. + fetchNextBatchAsyncIfPresent(); listingCount++; LOG.debug("New listing status: {}", this); } catch (AmazonClientException e) { throw translateException("listObjects()", listPath, e); } } - return objects; + // Storing the current result to be used by hasNext() call. + objectsPrev = objects; + return objectsPrev; + } + + /** + * If there are more listings present, call for next batch async. + * @throws IOException + */ + private void fetchNextBatchAsyncIfPresent() throws IOException { + if (objects.isTruncated()) { + LOG.debug("[{}], Requesting next {} objects under {}", + listingCount, maxKeys, listPath); + s3ListResultFuture = listingOperationCallbacks + .continueListObjectsAsync(request, objects); + } } @Override diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index ac9904a867e21..63c80bdd067e1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1649,19 +1649,21 @@ protected class ListingOperationCallbacksImpl implements @Override @Retries.RetryRaw - public S3ListResult listObjects( + public CompletableFuture listObjectsAsync( S3ListRequest request) throws IOException { - return S3AFileSystem.this.listObjects(request); + return submit(unboundedThreadPool, + () -> listObjects(request)); } @Override @Retries.RetryRaw - public S3ListResult continueListObjects( + public CompletableFuture continueListObjectsAsync( S3ListRequest request, S3ListResult prevResult) throws IOException { - return S3AFileSystem.this.continueListObjects(request, prevResult); + return submit(unboundedThreadPool, + () -> continueListObjects(request, prevResult)); } @Override @@ -2279,6 +2281,7 @@ public UploadInfo putObject(PutObjectRequest putObjectRequest) { * not be saved to the metadata store and * fs.s3a.metadatastore.fail.on.write.error=true */ + @VisibleForTesting @Retries.OnceRaw("For PUT; post-PUT actions are RetryTranslated") PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest) throws AmazonClientException, MetadataPersistenceException { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java index 5def31bc15c77..e5f9f7d9808ea 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a.impl; import java.io.IOException; +import java.util.concurrent.CompletableFuture; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -38,7 +39,7 @@ public interface ListingOperationCallbacks { /** - * Initiate a {@code listObjects} operation, incrementing metrics + * Initiate a {@code listObjectsAsync} operation, incrementing metrics * in the process. * * Retry policy: retry untranslated. @@ -47,7 +48,7 @@ public interface ListingOperationCallbacks { * @throws IOException if the retry invocation raises one (it shouldn't). */ @Retries.RetryRaw - S3ListResult listObjects( + CompletableFuture listObjectsAsync( S3ListRequest request) throws IOException; @@ -60,7 +61,7 @@ S3ListResult listObjects( * @throws IOException none, just there for retryUntranslated. */ @Retries.RetryRaw - S3ListResult continueListObjects( + CompletableFuture continueListObjectsAsync( S3ListRequest request, S3ListResult prevResult) throws IOException; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java index 7fa03a16cd199..0729f2ac289db 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -243,13 +244,14 @@ private StoreContext createMockStoreContext(boolean multiDelete, private static class MinimalListingOperationCallbacks implements ListingOperationCallbacks { @Override - public S3ListResult listObjects(S3ListRequest request) + public CompletableFuture listObjectsAsync( + S3ListRequest request) throws IOException { return null; } @Override - public S3ListResult continueListObjects( + public CompletableFuture continueListObjectsAsync( S3ListRequest request, S3ListResult prevResult) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java index a087dab6a5015..a3cca75c50c18 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java @@ -18,14 +18,33 @@ package org.apache.hadoop.fs.s3a.scale; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.Statistic; + import org.junit.Test; +import org.assertj.core.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.PutObjectResult; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; @@ -137,6 +156,93 @@ public void testListOperations() throws Throwable { } } + @Test + public void testMultiPagesListingPerformanceAndCorrectness() + throws Throwable { + describe("Check performance and correctness for multi page listing " + + "using different listing api"); + final Path dir = methodPath(); + final int batchSize = 10; + final int numOfPutRequests = 1000; + final int eachFileProcessingTime = 10; + final int numOfPutThreads = 50; + final Configuration conf = + getConfigurationWithConfiguredBatchSize(batchSize); + final InputStream im = new InputStream() { + @Override + public int read() throws IOException { + return -1; + } + }; + final List originalListOfFiles = new ArrayList<>(); + List> putObjectRequests = new ArrayList<>(); + ExecutorService executorService = Executors + .newFixedThreadPool(numOfPutThreads); + + NanoTimer uploadTimer = new NanoTimer(); + try(S3AFileSystem fs = (S3AFileSystem) FileSystem.get(dir.toUri(), conf)) { + fs.create(dir); + assume("Test is only for raw fs", !fs.hasMetadataStore()); + for (int i=0; i + fs.getWriteOperationHelper().putObject(put)); + } + executorService.invokeAll(putObjectRequests); + uploadTimer.end("uploading %d files with a parallelism of %d", + numOfPutRequests, numOfPutThreads); + + RemoteIterator resIterator = fs.listFiles(dir, true); + List listUsingListFiles = new ArrayList<>(); + NanoTimer timeUsingListFiles = new NanoTimer(); + while(resIterator.hasNext()) { + listUsingListFiles.add(resIterator.next().getPath().toString()); + Thread.sleep(eachFileProcessingTime); + } + timeUsingListFiles.end("listing %d files using listFiles() api with " + + "batch size of %d including %dms of processing time" + + " for each file", + numOfPutRequests, batchSize, eachFileProcessingTime); + + Assertions.assertThat(listUsingListFiles) + .describedAs("Listing results using listFiles() must" + + "match with original list of files") + .hasSameElementsAs(originalListOfFiles) + .hasSize(numOfPutRequests); + List listUsingListStatus = new ArrayList<>(); + NanoTimer timeUsingListStatus = new NanoTimer(); + FileStatus[] fileStatuses = fs.listStatus(dir); + for(FileStatus fileStatus : fileStatuses) { + listUsingListStatus.add(fileStatus.getPath().toString()); + Thread.sleep(eachFileProcessingTime); + } + timeUsingListStatus.end("listing %d files using listStatus() api with " + + "batch size of %d including %dms of processing time" + + " for each file", + numOfPutRequests, batchSize, eachFileProcessingTime); + Assertions.assertThat(listUsingListStatus) + .describedAs("Listing results using listStatus() must" + + "match with original list of files") + .hasSameElementsAs(originalListOfFiles) + .hasSize(numOfPutRequests); + } finally { + executorService.shutdown(); + } + } + + private Configuration getConfigurationWithConfiguredBatchSize(int batchSize) { + Configuration conf = new Configuration(getFileSystem().getConf()); + S3ATestUtils.disableFilesystemCaching(conf); + conf.setInt(Constants.MAX_PAGING_KEYS, batchSize); + return conf; + } + @Test public void testTimeToStatEmptyDirectory() throws Throwable { describe("Time to stat an empty directory"); @@ -188,5 +294,4 @@ private void timeToStatPath(Path path) throws IOException { LOG.info("listObjects: {}", listRequests); LOG.info("listObjects: per operation {}", listRequests.diff() / attempts); } - } From 82a750564631eb3077603bed27058006b3d66309 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Tue, 25 Aug 2020 13:29:12 +0200 Subject: [PATCH 099/335] YARN-10304. Create an endpoint for remote application log directory path query. Contributed by Andras Gyori --- .../mapreduce/v2/hs/webapp/HsWebServices.java | 18 +++ .../v2/hs/webapp/TestHsWebServicesLogs.java | 128 +++++++++++++++++- .../LogAggregationFileController.java | 8 ++ .../hadoop/yarn/server/webapp/LogServlet.java | 48 +++++++ .../server/webapp/YarnWebServiceParams.java | 1 + .../server/webapp/dao/RemoteLogPathEntry.java | 53 ++++++++ .../server/webapp/dao/RemoteLogPaths.java | 50 +++++++ 7 files changed, 305 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPathEntry.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPaths.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index 4ba8fe0b3773e..008edf5e57ee1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -423,6 +423,24 @@ public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters( return new JobTaskAttemptCounterInfo(ta); } + /** + * Returns the user qualified path name of the remote log directory for + * each pre-configured log aggregation file controller. + * + * @param req HttpServletRequest + * @return Path names grouped by file controller name + */ + @GET + @Path("/remote-log-dir") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response getRemoteLogDirPath(@Context HttpServletRequest req, + @QueryParam(YarnWebServiceParams.REMOTE_USER) String user, + @QueryParam(YarnWebServiceParams.APP_ID) String appIdStr) + throws IOException { + init(); + return logServlet.getRemoteLogDirPath(user, appIdStr); + } + @GET @Path("/aggregatedlogs") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java index 22aa3acd9a31f..2b43e240dcaf5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.hs.HistoryContext; import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; @@ -48,9 +49,13 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; +import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController; import org.apache.hadoop.yarn.server.webapp.LogServlet; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPathEntry; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPaths; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; @@ -68,6 +73,8 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -76,6 +83,8 @@ import java.util.stream.Collectors; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -116,6 +125,9 @@ public class TestHsWebServicesLogs extends JerseyTestBase { private static final String USER = "fakeUser"; private static final String FILE_NAME = "syslog"; + private static final String REMOTE_LOG_DIR_SUFFIX = "test-logs"; + private static final String[] FILE_FORMATS = {"IFile", "TFile"}; + private static final String NM_WEBADDRESS_1 = "test-nm-web-address-1:9999"; private static final NodeId NM_ID_1 = NodeId.newInstance("fakeHost1", 9951); private static final String NM_WEBADDRESS_2 = "test-nm-web-address-2:9999"; @@ -156,6 +168,17 @@ public class TestHsWebServicesLogs extends JerseyTestBase { } private static class WebServletModule extends ServletModule { + private Configuration newConf; + + WebServletModule() { + super(); + } + + WebServletModule(Configuration newConf) { + super(); + this.newConf = newConf; + } + @Override protected void configureServlets() { MockHistoryContext appContext = new MockHistoryContext(0, 1, 2, 1); @@ -199,8 +222,9 @@ protected void configureServlets() { fail("Failed to setup WebServletModule class"); } + Configuration usedConf = newConf == null ? conf : newConf; HsWebServices hsWebServices = - new HsWebServices(appContext, conf, webApp, mockProtocol); + new HsWebServices(appContext, usedConf, webApp, mockProtocol); try { LogServlet logServlet = hsWebServices.getLogServlet(); logServlet = spy(logServlet); @@ -576,6 +600,92 @@ public void testGetContainerLogFileForRunningContainer() throws Exception { + ContainerLogAggregationType.AGGREGATED, "Hello-" + CONTAINER_2_2_3); } + @Test + public void testRemoteLogDirWithUser() { + createReconfiguredServlet(); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1") + .path("history").path("remote-log-dir") + .queryParam(YarnWebServiceParams.REMOTE_USER, + USER) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + RemoteLogPaths res = response. + getEntity(new GenericType(){}); + + List collectedControllerNames = new ArrayList<>(); + for (RemoteLogPathEntry entry: res.getPaths()) { + String path = String.format("%s/%s/bucket-%s-%s", + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER, + REMOTE_LOG_DIR_SUFFIX, entry.getFileController().toLowerCase()); + collectedControllerNames.add(entry.getFileController()); + assertEquals(entry.getPath(), path); + } + + assertTrue(collectedControllerNames.containsAll( + Arrays.asList(FILE_FORMATS))); + } + + @Test + public void testRemoteLogDir() { + createReconfiguredServlet(); + UserGroupInformation ugi = UserGroupInformation. + createRemoteUser(USER); + UserGroupInformation.setLoginUser(ugi); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1") + .path("history").path("remote-log-dir") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + RemoteLogPaths res = response. + getEntity(new GenericType(){}); + + List collectedControllerNames = new ArrayList<>(); + for (RemoteLogPathEntry entry: res.getPaths()) { + String path = String.format("%s/%s/bucket-%s-%s", + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER, + REMOTE_LOG_DIR_SUFFIX, entry.getFileController().toLowerCase()); + collectedControllerNames.add(entry.getFileController()); + assertEquals(entry.getPath(), path); + } + + assertTrue(collectedControllerNames.containsAll( + Arrays.asList(FILE_FORMATS))); + } + + @Test + public void testRemoteLogDirWithUserAndAppId() { + createReconfiguredServlet(); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1") + .path("history").path("remote-log-dir") + .queryParam(YarnWebServiceParams.REMOTE_USER, + USER) + .queryParam(YarnWebServiceParams.APP_ID, + APPID_1.toString()) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + RemoteLogPaths res = response. + getEntity(new GenericType(){}); + + List collectedControllerNames = new ArrayList<>(); + for (RemoteLogPathEntry entry: res.getPaths()) { + String path = String.format("%s/%s/bucket-%s-%s/0001/%s", + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER, + REMOTE_LOG_DIR_SUFFIX, entry.getFileController().toLowerCase(), + APPID_1.toString()); + collectedControllerNames.add(entry.getFileController()); + assertEquals(entry.getPath(), path); + } + + assertTrue(collectedControllerNames.containsAll( + Arrays.asList(FILE_FORMATS))); + } + + @Test public void testNonExistingAppId() { ApplicationId nonExistingApp = ApplicationId.newInstance(99, 99); @@ -763,4 +873,20 @@ private static String getRedirectURL(String url) throws Exception { } return null; } + + private void createReconfiguredServlet() { + Configuration newConf = new YarnConfiguration(); + newConf.setStrings(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, + FILE_FORMATS); + newConf.setClass(String.format( + YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT, "IFile"), + LogAggregationIndexedFileController.class, + LogAggregationFileController.class); + newConf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR); + newConf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, + REMOTE_LOG_DIR_SUFFIX); + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule(newConf))); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 2bf5f4e6a4e99..9c609beb59a34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -157,6 +157,14 @@ public String getRemoteRootLogDirSuffix() { return this.remoteRootLogDirSuffix; } + /** + * Get the name of the file controller. + * @return name of the file controller. + */ + public String getFileControllerName() { + return this.fileControllerName; + } + /** * Initialize the writer. * @param context the {@link LogAggregationFileControllerContext} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index 33de8df01111e..b4f9a1f8982cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -24,13 +24,19 @@ import com.sun.jersey.api.client.UniformInterfaceException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; +import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPathEntry; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPaths; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -45,6 +51,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; +import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; @@ -174,6 +181,47 @@ private void validateUserInput(ApplicationId applicationId, } } + /** + * Returns the user qualified path name of the remote log directory for + * each pre-configured log aggregation file controller. + * + * @return {@link Response} object containing remote log dir path names + */ + public Response getRemoteLogDirPath(String user, String applicationId) + throws IOException { + String remoteUser = user; + ApplicationId appId = applicationId != null ? + ApplicationIdPBImpl.fromString(applicationId) : null; + + if (remoteUser == null) { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + remoteUser = ugi.getUserName(); + } + + List fileControllers = + getOrCreateFactory().getConfiguredLogAggregationFileControllerList(); + List paths = new ArrayList<>(); + + for (LogAggregationFileController fileController : fileControllers) { + String path; + if (appId != null) { + path = fileController.getRemoteAppLogDir(appId, remoteUser).toString(); + } else { + path = LogAggregationUtils.getRemoteLogSuffixedDir( + fileController.getRemoteRootLogDir(), + remoteUser, fileController.getRemoteRootLogDirSuffix()).toString(); + } + + paths.add(new RemoteLogPathEntry(fileController.getFileControllerName(), + path)); + } + + RemoteLogPaths result = new RemoteLogPaths(paths); + Response.ResponseBuilder response = Response.ok().entity(result); + response.header("X-Content-Type-Options", "nosniff"); + return response.build(); + } + public Response getLogsInfo(HttpServletRequest hsr, String appIdStr, String appAttemptIdStr, String containerIdStr, String nmId, boolean redirectedFromNode, boolean manualRedirection) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java index 0d9e9f68c1008..3aade3faafce8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -39,4 +39,5 @@ public interface YarnWebServiceParams { String REDIRECTED_FROM_NODE = "redirected_from_node"; String CLUSTER_ID = "clusterid"; String MANUAL_REDIRECTION = "manual_redirection"; + String REMOTE_USER = "user"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPathEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPathEntry.java new file mode 100644 index 0000000000000..76a1b8afa955d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPathEntry.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.webapp.dao; + +/** + * A remote log path for a log aggregation file controller. + * 
+ *   /%USER/
+ *
+ */ +public class RemoteLogPathEntry { + private String fileController; + private String path; + + //JAXB needs this + public RemoteLogPathEntry() {} + + public RemoteLogPathEntry(String fileController, String path) { + this.fileController = fileController; + this.path = path; + } + + public String getFileController() { + return fileController; + } + + public void setFileController(String fileController) { + this.fileController = fileController; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPaths.java new file mode 100644 index 0000000000000..80354fa0d6c49 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPaths.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +/** + * Container of a list of {@link RemoteLogPathEntry}. + */ +@XmlRootElement(name = "remoteLogDirPathResult") +@XmlAccessorType(XmlAccessType.FIELD) +public class RemoteLogPaths { + + @XmlElement(name = "paths") + private List paths; + + //JAXB needs this + public RemoteLogPaths() {} + + public RemoteLogPaths(List paths) { + this.paths = paths; + } + + public List getPaths() { + return paths; + } + + public void setPaths(List paths) { + this.paths = paths; + } +} \ No newline at end of file From 6a49bf9bffbe9bad1c719fc3a3b734f0060df70a Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 25 Aug 2020 15:18:36 +0100 Subject: [PATCH 100/335] HDFS-14852. Removing from LowRedundancyBlocks does not remove the block from all queues. Contributed by Fei Hui. --- .../hdfs/server/blockmanagement/BlockManager.java | 2 +- .../server/blockmanagement/LowRedundancyBlocks.java | 5 +++-- .../blockmanagement/TestLowRedundancyBlockQueues.java | 11 +++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f2cd6b9819efb..9ac7889cd3f38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2531,7 +2531,7 @@ void processPendingReconstructions() { * with the most up-to-date block information (e.g. genstamp). */ BlockInfo bi = blocksMap.getStoredBlock(timedOutItems[i]); - if (bi == null) { + if (bi == null || bi.isDeleted()) { continue; } NumberReplicas num = countNodes(timedOutItems[i]); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java index 8cf9dd40ca6d8..f6ef248e79f14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java @@ -382,17 +382,18 @@ boolean remove(BlockInfo block, int priLevel, int oldExpectedReplicas) { } else { // Try to remove the block from all queues if the block was // not found in the queue for the given priority level. + boolean found = false; for (int i = 0; i < LEVEL; i++) { if (i != priLevel && priorityQueues.get(i).remove(block)) { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.LowRedundancyBlock.remove: Removing block" + " {} from priority queue {}", block, i); decrementBlockStat(block, i, oldExpectedReplicas); - return true; + found = true; } } + return found; } - return false; } private void decrementBlockStat(BlockInfo blockInfo, int priLevel, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java index 000c636716c98..c40f277e3775c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java @@ -276,4 +276,15 @@ private void assertInLevel(LowRedundancyBlocks queues, } fail("Block " + block + " not found in level " + level); } + + @Test + public void testRemoveBlockInManyQueues() { + LowRedundancyBlocks neededReconstruction = new LowRedundancyBlocks(); + BlockInfo block = new BlockInfoContiguous(new Block(), (short)1024); + neededReconstruction.add(block, 2, 0, 1, 3); + neededReconstruction.add(block, 0, 0, 0, 3); + neededReconstruction.remove(block, LowRedundancyBlocks.LEVEL); + assertFalse("Should not contain the block.", + neededReconstruction.contains(block)); + } } From 6e618b6a7e3b7cb6459091945c8eb07fddc0034e Mon Sep 17 00:00:00 2001 From: hemanthboyina Date: Wed, 26 Aug 2020 13:03:08 +0530 Subject: [PATCH 101/335] HDFS-15536. RBF: Clear Quota in Router was not consistent. --- .../federation/store/records/MountTable.java | 5 ++ ...MultipleDestinationMountTableResolver.java | 49 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java index 282fe6cbb53e2..907a4055adb82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java @@ -430,6 +430,8 @@ public int hashCode() { .append(this.isReadOnly()) .append(this.getDestOrder()) .append(this.isFaultTolerant()) + .append(this.getQuota().getQuota()) + .append(this.getQuota().getSpaceQuota()) .toHashCode(); } @@ -443,6 +445,9 @@ public boolean equals(Object obj) { .append(this.isReadOnly(), other.isReadOnly()) .append(this.getDestOrder(), other.getDestOrder()) .append(this.isFaultTolerant(), other.isFaultTolerant()) + .append(this.getQuota().getQuota(), other.getQuota().getQuota()) + .append(this.getQuota().getSpaceQuota(), + other.getQuota().getSpaceQuota()) .isEquals(); } return false; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java index 181442d64708b..6ebc31109d9fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java @@ -25,6 +25,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -34,6 +35,7 @@ import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options.Rename; @@ -57,7 +59,9 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDestinationResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.hdfs.tools.federation.RouterAdmin; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.ToolRunner; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -542,6 +546,43 @@ public void testRenameMultipleDestDirectories() throws Exception { verifyRenameOnMultiDestDirectories(DestinationOrder.SPACE, true); } + @Test + public void testClearQuota() throws Exception { + long nsQuota = 5; + long ssQuota = 100; + Path path = new Path("/router_test"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + MountTable addEntry = MountTable.newInstance("/router_test", + Collections.singletonMap("ns0", "/router_test")); + addEntry.setQuota(new RouterQuotaUsage.Builder().build()); + assertTrue(addMountTable(addEntry)); + RouterQuotaUpdateService updateService = + routerContext.getRouter().getQuotaCacheUpdateService(); + updateService.periodicInvoke(); + + //set quota and validate the quota + RouterAdmin admin = getRouterAdmin(); + String[] argv = new String[] {"-setQuota", path.toString(), "-nsQuota", + String.valueOf(nsQuota), "-ssQuota", String.valueOf(ssQuota)}; + assertEquals(0, ToolRunner.run(admin, argv)); + updateService.periodicInvoke(); + resolver.loadCache(true); + ContentSummary cs = routerFs.getContentSummary(path); + assertEquals(nsQuota, cs.getQuota()); + assertEquals(ssQuota, cs.getSpaceQuota()); + + //clear quota and validate the quota + argv = new String[] {"-clrQuota", path.toString()}; + assertEquals(0, ToolRunner.run(admin, argv)); + updateService.periodicInvoke(); + resolver.loadCache(true); + //quota should be cleared + ContentSummary cs1 = routerFs.getContentSummary(path); + assertEquals(-1, cs1.getQuota()); + assertEquals(-1, cs1.getSpaceQuota()); + } + /** * Test to verify rename operation on directories in case of multiple * destinations. @@ -723,4 +764,12 @@ private static FileSystem getFileSystem(final String nsId) { return null; } + private RouterAdmin getRouterAdmin() { + Router router = routerContext.getRouter(); + Configuration configuration = routerContext.getConf(); + InetSocketAddress routerSocket = router.getAdminServerAddress(); + configuration.setSocketAddr(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, + routerSocket); + return new RouterAdmin(configuration); + } } \ No newline at end of file From 75db5526b5db2675a7f396715f48733a7ed26acf Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 26 Aug 2020 13:08:14 +0530 Subject: [PATCH 102/335] YARN-1806. Add ThreadDump Option in YARN UI2 to fetch for running containers Contributed by Siddharth Ahuja. Reviewed by Akhil PB. --- .../app/adapters/yarn-container-threaddump.js | 88 ++++++ .../app/adapters/yarn-node-container-log.js | 94 ++++++ .../app/controllers/yarn-app/threaddump.js | 281 ++++++++++++++++++ .../src/main/webapp/app/mixins/app-attempt.js | 22 ++ .../app/models/yarn-node-container-log.js | 23 ++ .../src/main/webapp/app/router.js | 1 + .../webapp/app/routes/yarn-app/threaddump.js | 68 +++++ .../serializers/yarn-node-container-log.js | 50 ++++ .../main/webapp/app/templates/yarn-app.hbs | 3 + .../app/templates/yarn-app/threaddump.hbs | 113 +++++++ 10 files changed, 743 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-container-threaddump.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-node-container-log.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/threaddump.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-node-container-log.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/threaddump.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container-log.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/threaddump.hbs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-container-threaddump.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-container-threaddump.js new file mode 100644 index 0000000000000..c4e9382b9b859 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-container-threaddump.js @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import RESTAbstractAdapter from './restabstract'; + +export default RESTAbstractAdapter.extend({ + address: "rmWebAddress", + restNameSpace: "cluster", + + handleResponse(status, headers, payload, requestData) { + // If the user is not authorized to signal a threaddump for a container, + // the response contains a RemoteException with a 403 (Forbidden) status + // code. Extract out the error message from the RemoteException in this + // case. + // If the status is '0' or empty, it is symptomatic of the YARN role not + // available or able to respond or a network timeout/firewall issue. + if (status === 403) { + if (payload + && typeof payload === 'object' + && payload.RemoteException + && payload.RemoteException.message) { + return new Error(payload.RemoteException.message); + } + } else if (status === 0 && payload === "") { + return new Error("Not able to connect to YARN!"); + } + + return payload; + }, + + /** + * Set up the POST request to use the signalToContainer REST API + * to signal a thread dump for a running container to RM. + */ + signalContainerForThreaddump(request, containerId, user) { + var url = this.buildURL(); + if (user && containerId) { + url += "/containers" + "/" + containerId + "/signal" + + "/OUTPUT_THREAD_DUMP" + "?user.name=" + user; + } + return this.ajax(url, "POST", {data: request}); + }, + + ajax(url, method, hash) { + hash = {}; + hash.crossDomain = true; + hash.xhrFields = {withCredentials: true}; + hash.targetServer = "RM"; + return this._super(url, method, hash); + }, + + /** + * Override options so that result is not expected to be JSON + */ + ajaxOptions: function (url, type, options) { + var hash = options || {}; + hash.url = url; + hash.type = type; + // Make sure jQuery does not try to convert response to JSON. + hash.dataType = 'text'; + hash.context = this; + + var headers = Ember.get(this, 'headers'); + if (headers !== undefined) { + hash.beforeSend = function (xhr) { + Object.keys(headers).forEach(function (key) { + return xhr.setRequestHeader(key, headers[key]); + }); + }; + } + return hash; + } +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-node-container-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-node-container-log.js new file mode 100644 index 0000000000000..ba01a7037d694 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-node-container-log.js @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DS from 'ember-data'; +import Ember from 'ember'; +import Converter from 'yarn-ui/utils/converter'; + +/** + * REST URL's response when fetching container logs will be + * in plain text format and not JSON. + */ +export default DS.RESTAdapter.extend({ + headers: { + Accept: 'text/plain' + }, + + host: Ember.computed("address", function () { + return this.get(`hosts.localBaseAddress`); + }), + + namespace: Ember.computed("restNameSpace", function () { + return this.get(`env.app.namespaces.node`); + }), + + urlForQueryRecord(query) { + var url = this._buildURL(); + url = url.replace("{nodeAddress}", query.nodeHttpAddr) + "/containerlogs/" + + query.containerId + "/" + query.fileName; + return url; + }, + + queryRecord: function (store, type, query) { + var url = this.urlForQueryRecord(query); + // Query params not required. + query = null; + console.log(url); + return this.ajax(url, 'GET', { data: query }); + }, + + ajax(url, method, hash) { + hash = hash || {}; + hash.crossDomain = true; + hash.xhrFields = {withCredentials: true}; + hash.targetServer = "NM"; + return this._super(url, method, hash); + }, + + /** + * Override options so that result is not expected to be JSON + */ + ajaxOptions: function (url, type, options) { + var hash = options || {}; + hash.url = url; + hash.type = type; + // Make sure jQuery does not try to convert response to JSON. + hash.dataType = 'text'; + hash.context = this; + + var headers = Ember.get(this, 'headers'); + if (headers !== undefined) { + hash.beforeSend = function (xhr) { + Object.keys(headers).forEach(function (key) { + return xhr.setRequestHeader(key, headers[key]); + }); + }; + } + return hash; + }, + + handleResponse(status, headers, payload, requestData) { + // If the status is '0' or empty, it is symptomatic of the YARN role not + // available or able to respond or a network timeout/firewall issue. + if (status === 0 && payload === "") { + return new Error("Not able to connect to YARN!"); + } + + return payload; + } +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/threaddump.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/threaddump.js new file mode 100644 index 0000000000000..4b1e57b5ec8c4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/threaddump.js @@ -0,0 +1,281 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; +import Constants from 'yarn-ui/constants'; + +export default Ember.Controller.extend({ + queryParams: ["service", "attempt", "containerid"], + service: undefined, + attempt: undefined, + containerid: undefined, + + selectedAttemptId: "", + attemptContainerList: null, + selectedContainerId: "", + selectedLogFileName: "", + containerLogFiles: null, + selectedContainerThreaddumpContent: "", + containerNodeMappings: [], + threaddumpErrorMessage: "", + stdoutLogsFetchFailedError: "", + appAttemptToStateMappings: [], + + _isLoadingTopPanel: false, + _isLoadingBottomPanel: false, + _isThreaddumpAttemptFailed: false, + _isStdoutLogsFetchFailed: false, + + initializeSelect: function(selector = ".js-fetch-attempt-containers") { + Ember.run.schedule("afterRender", this, function() { + $(selector).select2({ width: "350px", multiple: false }); + }); + }, + + actions: { + showContainersForAttemptId(attemptId, containerId = "") { + this.set("selectedAttemptId", ""); + if (attemptId) { + this.set("_isLoadingTopPanel", true); + this.set("selectedAttemptId", attemptId); + this.fetchRunningContainersForAttemptId(attemptId) + .then(hash => { + let containers = null; + let containerIdArr = {}; + var containerNodeMapping = null; + var nodeHttpAddress = null; + + // Getting RUNNING containers from the RM first. + if ( + hash.rmContainers.get("length") > 0 && + hash.rmContainers.get("content") + ) { + // Create a container-to-NMnode mapping for later use. + hash.rmContainers.get("content").forEach( + function(containerInfo) { + nodeHttpAddress = containerInfo._data.nodeHttpAddress; + nodeHttpAddress = nodeHttpAddress + .replace(/(^\w+:|^)\/\//, ''); + containerNodeMapping = Ember.Object.create({ + name: containerInfo.id, + value: nodeHttpAddress + }); + this.containerNodeMappings.push(containerNodeMapping); + containerIdArr[containerInfo.id] = true; + }.bind(this)); + + containers = (containers || []).concat( + hash.rmContainers.get("content") + ); + } + + this.set("attemptContainerList", containers); + this.initializeSelect(".js-fetch-threaddump-containers"); + + if (containerId) { + this.send("showThreaddumpForContainer", containerId); + } + }) + .finally(() => { + this.set("_isLoadingTopPanel", false); + }); + } else { + this.set("attemptContainerList", null); + this.set("selectedContainerId", ""); + this.set("containerLogFiles", null); + this.set("selectedLogFileName", ""); + this.set("selectedContainerThreaddumpContent", ""); + this.set("containerNodeMappings", []); + } + }, + + showThreaddumpForContainer(containerIdForThreaddump) { + Ember.$("#logContentTextArea1234554321").val(""); + this.set("showFullThreaddump", false); + this.set("selectedContainerId", containerIdForThreaddump); + this.set("_isLoadingBottomPanel", true); + this.set("_isStdoutLogsFetchFailed", false); + this.set("stdoutLogsFetchFailedError", ""); + this.set("_isThreaddumpAttemptFailed", false); + this.set("threaddumpErrorMessage", ""); + + if (containerIdForThreaddump) { + this.set("selectedContainerThreaddumpContent", ""); + + this.fetchThreaddumpForContainer(containerIdForThreaddump) + .then(function() { + Ember.Logger.log("Threaddump attempt has been successful!"); + + var currentContainerId = null; + var currentContainerNode = null; + var nodeForContainerThreaddump = null; + + // Fetch the NM node for the selected container from the + // mappings stored above when + this.containerNodeMappings.forEach(function(item) { + currentContainerId = item.name; + currentContainerNode = item.value; + + if ((currentContainerId === containerIdForThreaddump) + && nodeForContainerThreaddump == null) { + nodeForContainerThreaddump = currentContainerNode; + } + }); + + if (nodeForContainerThreaddump) { + // Fetch stdout logs after 1.2 seconds of a successful POST + // request to RM. This is to allow for sufficient time for the NM + // to heartbeat into RM (default of 1 second) whereupon it will + // receive RM's signal to post a threaddump that will be captured + // in the stdout logs of the container. The extra 200ms is to + // allow for any network/IO delays. + Ember.run.later((function() { + this.fetchLogsForContainer(nodeForContainerThreaddump, + containerIdForThreaddump, + "stdout") + .then( + hash => { + this.set("selectedContainerThreaddumpContent", + hash.logs.get('logs').trim()); + }.bind(this)) + .catch(function(error) { + this.set("_isStdoutLogsFetchFailed", true); + this.set("stdoutLogsFetchFailedError", error); + }.bind(this)) + .finally(function() { + this.set("_isLoadingBottomPanel", false); + }.bind(this)); + }.bind(this)), 1200); + } + }.bind(this), function(error) { + this.set("_isThreaddumpAttemptFailed", true); + this.set("threaddumpErrorMessage", error); + this.set("_isLoadingBottomPanel", false); + }.bind(this)).finally(function() { + this.set("selectedContainerThreaddumpContent", ""); + }.bind(this)); + } else { + this.set("selectedContainerId", ""); + this.set("selectedContainerThreaddumpContent", ""); + } + } + }, + + // Set up the running attempt for the first time threaddump button is clicked. + runningAttempt: Ember.computed("model.attempts", function() { + let attempts = this.get("model.attempts"); + let runningAttemptId = null; + + if (attempts && attempts.get("length") && attempts.get("content")) { + attempts.get("content").forEach(function(appAttempt) { + if (appAttempt._data.state === "RUNNING") { + runningAttemptId = appAttempt._data.appAttemptId; + } + }); + } + + return runningAttemptId; + }), + + /** + * Create and send fetch running containers request. + */ + fetchRunningContainersForAttemptId(attemptId) { + let request = {}; + + request["rmContainers"] = this.store + .query("yarn-container", { + app_attempt_id: attemptId + }) + .catch(function(error) { + return Ember.A(); + }); + + return Ember.RSVP.hash(request); + }, + + /** + * Create and send fetch logs request for a selected container, from a + * specific node. + */ + fetchLogsForContainer(nodeForContainer, containerId, logFile) { + let request = {}; + + request["logs"] = this.store + .queryRecord("yarn-node-container-log", { + nodeHttpAddr: nodeForContainer, + containerId: containerId, + fileName: logFile + }) + + return Ember.RSVP.hash(request); + }, + + /** + * Send out a POST request to RM to signal a threaddump for the selected + * container for the currently logged-in user. + */ + fetchThreaddumpForContainer(containerId) { + var adapter = this.store.adapterFor("yarn-container-threaddump"); + + let requestedUser = ""; + if (this.model + && this.model.userInfo + && this.model.userInfo.get('firstObject')) { + requestedUser = this.model.userInfo + .get('firstObject') + .get('requestedUser'); + } + + return adapter.signalContainerForThreaddump({}, containerId, requestedUser); + }, + + /** + * Reset attributes after a refresh. + */ + resetAfterRefresh() { + this.set("selectedAttemptId", ""); + this.set("attemptContainerList", null); + this.set("selectedContainerId", ""); + this.set("selectedLogFileName", ""); + this.set("containerLogFiles", null); + this.set("selectedContainerThreaddumpContent", ""); + this.set("containerNodeMappings", []); + this.set("_isThreaddumpAttemptFailed", false); + this.set("threaddumpErrorMessage", ""); + this.set("_isStdoutLogsFetchFailed", false); + this.set("stdoutLogsFetchFailedError", ""); + this.set("appAttemptToStateMappings", []); + }, + + // Set the threaddump content in the content area. + showThreaddumpContent: Ember.computed( + "selectedContainerThreaddumpContent", + function() { + return this.get("selectedContainerThreaddumpContent"); + } + ), + + /** + * Check if the application for the container whose threaddump is being attempted + * is even running. + */ + isRunningApp: Ember.computed('model.app.state', function() { + return this.get('model.app.state') === "RUNNING"; + }) +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/mixins/app-attempt.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/mixins/app-attempt.js index a90bf36fd9aaf..a1b9a2f0a24a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/mixins/app-attempt.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/mixins/app-attempt.js @@ -34,6 +34,17 @@ export default Ember.Mixin.create({ }); }, + fetchAppInfoFromRM(appId, store) { + return new Ember.RSVP.Promise(function(resolve, reject) { + store.find('yarn-app', appId).then(function(rmApp) { + resolve(rmApp); + }, function() { + console.error('Error:', 'Application not found in RM for appId: ' + appId); + reject(null); + }); + }); + }, + fetchAttemptInfoFromRMorATS(attemptId, store) { return new Ember.RSVP.Promise(function(resolve, reject) { store.findRecord('yarn-app-attempt', attemptId, {reload: true}).then(function(rmAttempt) { @@ -62,5 +73,16 @@ export default Ember.Mixin.create({ }); }); }); + }, + + fetchAttemptListFromRM(appId, store) { + return new Ember.RSVP.Promise(function(resolve, reject) { + store.query('yarn-app-attempt', {appId: appId}).then(function(rmAttempts) { + resolve(rmAttempts); + }, function() { + console.error('Error:', 'Application attempts not found in RM for appId: ' + appId); + reject(null); + }); + }); } }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-node-container-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-node-container-log.js new file mode 100644 index 0000000000000..01dc6db1cb562 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-node-container-log.js @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DS from 'ember-data'; + +export default DS.Model.extend({ + logs: DS.attr('string') +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js index 477bfc923e6f2..48a2e1b3854c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js @@ -63,6 +63,7 @@ Router.map(function() { this.route('charts'); this.route('configs'); this.route('logs'); + this.route('threaddump'); }); this.route('yarn-component-instances', function() { this.route('info', {path: '/:component_name/info'}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/threaddump.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/threaddump.js new file mode 100644 index 0000000000000..676c9d254ad9a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/threaddump.js @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; +import AbstractRoute from '../abstract'; +import AppAttemptMixin from 'yarn-ui/mixins/app-attempt'; + +export default AbstractRoute.extend(AppAttemptMixin, { + model(param, transition) { + const { app_id } = this.paramsFor('yarn-app'); + const { service } = param; + transition.send('updateBreadcrumbs', app_id, service, [{text: 'Threaddump'}]); + return Ember.RSVP.hash({ + appId: app_id, + serviceName: service, + attempts: this.fetchAttemptListFromRM(app_id, this.store) + .catch(function(error) { + Ember.Logger.log("App attempt list fetch failed!"); + Ember.Logger.log(error); + return []; + }), + app: this.fetchAppInfoFromRM(app_id, this.store), + userInfo: this.store.findAll('cluster-user-info', {reload: true}) + .catch(function(error) { + Ember.Logger.log("userInfo querying failed"); + Ember.Logger.log(error); + return null; + }) + }); + }, + + activate() { + const controller = this.controllerFor("yarn-app.threaddump"); + const { attempt, containerid } = this.paramsFor('yarn-app.threaddump'); + controller.resetAfterRefresh(); + controller.initializeSelect(); + if (attempt) { + controller.send("showContainersForAttemptId", attempt, containerid); + } else { + controller.set("selectedAttemptId", ""); + } + }, + + unloadAll() { + this.store.unloadAll('yarn-app-attempt'); + this.store.unloadAll('yarn-container'); + this.store.unloadAll('yarn-node-container-log'); + this.store.unloadAll('cluster-user-info'); + if (this.controller) { + this.controller.resetAfterRefresh(); + } + } +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container-log.js new file mode 100644 index 0000000000000..6f75a3045577c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container-log.js @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import DS from 'ember-data'; + +export default DS.JSONAPISerializer.extend({ + + internalNormalizeSingleResponse(store, primaryModelClass, payload) { + var fixedPayload = { + id: "yarn_node_container_log" + "_" + Date.now(), + type: primaryModelClass.modelName, + attributes: { + logs: payload + } + }; + return fixedPayload; + }, + + normalizeSingleResponse(store, primaryModelClass, payload/*, id, requestType*/) { + var p = this.internalNormalizeSingleResponse(store, + primaryModelClass, payload); + + return { data: p }; + }, + + normalizeArrayResponse(store, primaryModelClass, payload/*, id, requestType*/) { + var normalizedArrayResponse = { + data: [] + }; + + if (payload) { + normalizedArrayResponse.data = [this.internalNormalizeSingleResponse(store, primaryModelClass, payload)]; + } + return normalizedArrayResponse; + } +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs index c16a7a5f7c909..f6421d4492cfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app.hbs @@ -163,6 +163,9 @@ {{#link-to 'yarn-app.logs' tagName="li" class=(if (eq target.currentPath 'yarn-app.logs') "active")}} {{#link-to 'yarn-app.logs' appId (query-params service=serviceName)}}Logs{{/link-to}} {{/link-to}} + {{#link-to 'yarn-app.threaddump' tagName="li" class=(if (eq target.currentPath 'yarn-app.threaddump') "active")}} + {{#link-to 'yarn-app.threaddump' appId (query-params service=serviceName)}}Threaddump{{/link-to}} + {{/link-to}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/threaddump.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/threaddump.hbs new file mode 100644 index 0000000000000..4f7cfaaa24175 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/threaddump.hbs @@ -0,0 +1,113 @@ +{{! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +}} + +
+
+
+
+ Threaddump {{collapsible-panel targetId="threaddumpFilesCollapsablePanel"}} +
+
+ {{#if _isLoadingTopPanel}} +
+ Loading... +
+ {{/if}} + {{#unless isRunningApp}} +
+
+ + Threaddump cannot be collected for an application that is not running. +
+
+ {{else}} + {{#if runningAttempt}} +
+
+ +
+ +
+
+ {{#if attemptContainerList}} +
+ +
+ +
+
+ {{else}} + {{#if (and selectedAttemptId (not _isLoadingTopPanel))}} +
+

No container data available!

+
+ {{/if}} + {{/if}} +
+ {{else}} +
+

No data available!

+
+ {{/if}} + {{/unless}} +
+
+
+
+{{#if selectedContainerId}} +
+
+
+
+ Threaddump: [ {{selectedContainerId}} & {{selectedAttemptId}} ] + {{collapsible-panel targetId="threaddumpContentCollapsablePanel"}} +
+
+ {{#if _isLoadingBottomPanel}} +
+ Loading... +
+ {{/if}} +
+ {{#if _isThreaddumpAttemptFailed}} +
+ + Threaddump fetch failed for container: {{selectedContainerId}} due to: “{{threaddumpErrorMessage}}” +
+ {{else if _isStdoutLogsFetchFailed}} +
+ + Logs fetch failed for container: {{selectedContainerId}} due to: “{{stdoutLogsFetchFailedError}}” +
+ {{else}} + 
{{showThreaddumpContent}}
+ {{/if}} +
+
+
+
+
+{{/if}} From 931adbaa1412dcf8dc8679a8305c5146a4fe2821 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Wed, 26 Aug 2020 23:15:24 +0900 Subject: [PATCH 103/335] HADOOP-17224. Install Intel ISA-L library in Dockerfile. (#2243) --- dev-support/docker/Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 2af73eb8b75b0..c4c6bbfe4ecb3 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -167,6 +167,26 @@ RUN curl -L -s -S \ && shasum -a 512 /bin/hadolint | \ awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' +###### +# Intel ISA-L 2.29.0 +###### +# hadolint ignore=DL3003,DL3008 +RUN mkdir -p /opt/isa-l-src \ + && apt-get -q update \ + && apt-get install -y --no-install-recommends automake yasm \ + && apt-get clean \ + && curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz \ + && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ + && cd /opt/isa-l-src \ + && ./autogen.sh \ + && ./configure \ + && make \ + && make install \ + && cd /root \ + && rm -rf /opt/isa-l-src + ### # Avoid out of memory errors in builds ### From 5e52955112a3151bb608e092f31fc5084de78705 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Wed, 26 Aug 2020 10:41:10 -0700 Subject: [PATCH 104/335] Revert "HADOOP-17159 Ability for forceful relogin in UserGroupInformation class (#2197)" This reverts commit a932796d0cad3d84df0003782e4247cbc2dcca93. --- .../hadoop/security/UserGroupInformation.java | 35 ++++-------------- .../security/TestUGILoginFromKeytab.java | 36 ------------------- 2 files changed, 7 insertions(+), 64 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index d1ab4365a045e..91b64ade598e5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1232,26 +1232,7 @@ public void reloginFromKeytab() throws IOException { reloginFromKeytab(false); } - /** - * Force re-Login a user in from a keytab file. Loads a user identity from a - * keytab file and logs them in. They become the currently logged-in user. - * This method assumes that {@link #loginUserFromKeytab(String, String)} had - * happened already. The Subject field of this UserGroupInformation object is - * updated to have the new credentials. - * - * @param ignoreTimeElapsed Force re-login irrespective of the time of last - * login - * @throws IOException - * @throws KerberosAuthException on a failure - */ - @InterfaceAudience.Public - @InterfaceStability.Evolving - public void reloginFromKeytab(boolean ignoreTimeElapsed) throws IOException { - reloginFromKeytab(false, ignoreTimeElapsed); - } - - private void reloginFromKeytab(boolean checkTGT, boolean ignoreTimeElapsed) - throws IOException { + private void reloginFromKeytab(boolean checkTGT) throws IOException { if (!shouldRelogin() || !isFromKeytab()) { return; } @@ -1266,7 +1247,7 @@ private void reloginFromKeytab(boolean checkTGT, boolean ignoreTimeElapsed) return; } } - relogin(login, ignoreTimeElapsed); + relogin(login); } /** @@ -1287,27 +1268,25 @@ public void reloginFromTicketCache() throws IOException { if (login == null) { throw new KerberosAuthException(MUST_FIRST_LOGIN); } - relogin(login, false); + relogin(login); } - private void relogin(HadoopLoginContext login, boolean ignoreTimeElapsed) - throws IOException { + private void relogin(HadoopLoginContext login) throws IOException { // ensure the relogin is atomic to avoid leaving credentials in an // inconsistent state. prevents other ugi instances, SASL, and SPNEGO // from accessing or altering credentials during the relogin. synchronized(login.getSubjectLock()) { // another racing thread may have beat us to the relogin. if (login == getLogin()) { - unprotectedRelogin(login, ignoreTimeElapsed); + unprotectedRelogin(login); } } } - private void unprotectedRelogin(HadoopLoginContext login, - boolean ignoreTimeElapsed) throws IOException { + private void unprotectedRelogin(HadoopLoginContext login) throws IOException { assert Thread.holdsLock(login.getSubjectLock()); long now = Time.now(); - if (!hasSufficientTimeElapsed(now) && !ignoreTimeElapsed) { + if (!hasSufficientTimeElapsed(now)) { return; } // register most recent relogin attempt diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java index bf4cf75ba8179..d233234c26c0c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java @@ -158,42 +158,6 @@ public void testUGIReLoginFromKeytab() throws Exception { Assert.assertNotSame(login1, login2); } - /** - * Force re-login from keytab using the MiniKDC and verify the UGI can - * successfully relogin from keytab as well. - */ - @Test - public void testUGIForceReLoginFromKeytab() throws Exception { - // Set this to false as we are testing force re-login anyways - UserGroupInformation.setShouldRenewImmediatelyForTests(false); - String principal = "foo"; - File keytab = new File(workDir, "foo.keytab"); - kdc.createPrincipal(keytab, principal); - - UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); - UserGroupInformation ugi = UserGroupInformation.getLoginUser(); - Assert.assertTrue("UGI should be configured to login from keytab", - ugi.isFromKeytab()); - - // Verify relogin from keytab. - User user = getUser(ugi.getSubject()); - final long firstLogin = user.getLastLogin(); - final LoginContext login1 = user.getLogin(); - Assert.assertNotNull(login1); - - // Sleep for 2 secs to have a difference between first and second login - Thread.sleep(2000); - - // Force relogin from keytab - ugi.reloginFromKeytab(true); - final long secondLogin = user.getLastLogin(); - final LoginContext login2 = user.getLogin(); - Assert.assertTrue("User should have been able to relogin from keytab", - secondLogin > firstLogin); - Assert.assertNotNull(login2); - Assert.assertNotSame(login1, login2); - } - @Test public void testGetUGIFromKnownSubject() throws Exception { KerberosPrincipal principal = new KerberosPrincipal("user"); From ca8e7a77256003e11ab7e3d079ee4cf9f50080dd Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Thu, 27 Aug 2020 12:10:39 +0900 Subject: [PATCH 105/335] HDFS-15510. RBF: Quota and Content Summary was not correct in Multiple Destinations. Contributed by Hemanth Boyina. --- .../router/RouterClientProtocol.java | 6 +- ...MultipleDestinationMountTableResolver.java | 57 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index e2ec0303333f0..fb3eb18178361 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -1863,6 +1863,8 @@ private RemoteLocation getFirstMatchingLocation(RemoteLocation location, /** * Aggregate content summaries for each subcluster. + * If the mount point has multiple destinations + * add the quota set value only once. * * @param summaries Collection of individual summaries. * @return Aggregated content summary. @@ -1885,9 +1887,9 @@ private ContentSummary aggregateContentSummary( length += summary.getLength(); fileCount += summary.getFileCount(); directoryCount += summary.getDirectoryCount(); - quota += summary.getQuota(); + quota = summary.getQuota(); spaceConsumed += summary.getSpaceConsumed(); - spaceQuota += summary.getSpaceQuota(); + spaceQuota = summary.getSpaceQuota(); // We return from the first response as we assume that the EC policy // of each sub-cluster is same. if (ecPolicy.isEmpty()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java index 6ebc31109d9fd..ebb62d410defe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java @@ -583,6 +583,63 @@ public void testClearQuota() throws Exception { assertEquals(-1, cs1.getSpaceQuota()); } + @Test + public void testContentSummaryWithMultipleDest() throws Exception { + MountTable addEntry; + long nsQuota = 5; + long ssQuota = 100; + Path path = new Path("/testContentSummaryWithMultipleDest"); + Map destMap = new HashMap<>(); + destMap.put("ns0", "/testContentSummaryWithMultipleDest"); + destMap.put("ns1", "/testContentSummaryWithMultipleDest"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + addEntry = + MountTable.newInstance("/testContentSummaryWithMultipleDest", destMap); + addEntry.setQuota( + new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota) + .build()); + assertTrue(addMountTable(addEntry)); + RouterQuotaUpdateService updateService = + routerContext.getRouter().getQuotaCacheUpdateService(); + updateService.periodicInvoke(); + ContentSummary cs = routerFs.getContentSummary(path); + assertEquals(nsQuota, cs.getQuota()); + assertEquals(ssQuota, cs.getSpaceQuota()); + ContentSummary ns0Cs = nnFs0.getContentSummary(path); + assertEquals(nsQuota, ns0Cs.getQuota()); + assertEquals(ssQuota, ns0Cs.getSpaceQuota()); + ContentSummary ns1Cs = nnFs1.getContentSummary(path); + assertEquals(nsQuota, ns1Cs.getQuota()); + assertEquals(ssQuota, ns1Cs.getSpaceQuota()); + } + + @Test + public void testContentSummaryMultipleDestWithMaxValue() + throws Exception { + MountTable addEntry; + long nsQuota = Long.MAX_VALUE - 2; + long ssQuota = Long.MAX_VALUE - 2; + Path path = new Path("/testContentSummaryMultipleDestWithMaxValue"); + Map destMap = new HashMap<>(); + destMap.put("ns0", "/testContentSummaryMultipleDestWithMaxValue"); + destMap.put("ns1", "/testContentSummaryMultipleDestWithMaxValue"); + nnFs0.mkdirs(path); + nnFs1.mkdirs(path); + addEntry = MountTable + .newInstance("/testContentSummaryMultipleDestWithMaxValue", destMap); + addEntry.setQuota( + new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota) + .build()); + assertTrue(addMountTable(addEntry)); + RouterQuotaUpdateService updateService = + routerContext.getRouter().getQuotaCacheUpdateService(); + updateService.periodicInvoke(); + ContentSummary cs = routerFs.getContentSummary(path); + assertEquals(nsQuota, cs.getQuota()); + assertEquals(ssQuota, cs.getSpaceQuota()); + } + /** * Test to verify rename operation on directories in case of multiple * destinations. From 2ffe00fc46aa74929e722dc1804fb0b3d48ee7a9 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 26 Aug 2020 23:04:56 -0700 Subject: [PATCH 106/335] HDFS-15540. Directories protected from delete can still be moved to the trash. Contributed by Stephen O'Donnell. Signed-off-by: Wei-Chiu Chuang --- .../hdfs/server/namenode/FSDirRenameOp.java | 5 ++ .../namenode/TestProtectedDirectories.java | 70 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java index 7396519e90af2..43dd1b166a7a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java @@ -263,6 +263,11 @@ static RenameResult renameTo(FSDirectory fsd, FSPermissionChecker pc, throws IOException { final INodesInPath srcIIP = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); final INodesInPath dstIIP = fsd.resolvePath(pc, dst, DirOp.CREATE_LINK); + + if(fsd.isNonEmptyDirectory(srcIIP)) { + DFSUtil.checkProtectedDescendants(fsd, srcIIP); + } + if (fsd.isPermissionEnabled()) { boolean renameToTrash = false; if (null != options && diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java index c15af553fdbf1..e5f263155eb04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java @@ -26,6 +26,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.security.AccessControlException; @@ -36,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.*; @@ -284,6 +287,31 @@ public void testDelete() throws Throwable { } } + @Test + public void testMoveToTrash() throws Throwable { + for (TestMatrixEntry testMatrixEntry : createTestMatrix()) { + Configuration conf = new HdfsConfiguration(); + conf.setInt(DFSConfigKeys.FS_TRASH_INTERVAL_KEY, 3600); + MiniDFSCluster cluster = setupTestCase( + conf, testMatrixEntry.getProtectedPaths(), + testMatrixEntry.getUnprotectedPaths()); + + try { + LOG.info("Running {}", testMatrixEntry); + FileSystem fs = cluster.getFileSystem(); + for (Path path : testMatrixEntry.getAllPathsToBeDeleted()) { + assertThat( + testMatrixEntry + ": Testing whether " + path + + " can be moved to trash", + moveToTrash(fs, path, conf), + is(testMatrixEntry.canPathBeDeleted(path))); + } + } finally { + cluster.shutdown(); + } + } + } + /* * Verify that protected directories could not be renamed. */ @@ -339,6 +367,33 @@ public void testRenameProtectSubDirs() throws Throwable { } } + @Test + public void testMoveProtectedSubDirsToTrash() throws Throwable { + for (TestMatrixEntry testMatrixEntry : + createTestMatrixForProtectSubDirs()) { + Configuration conf = new HdfsConfiguration(); + conf.setBoolean(DFS_PROTECTED_SUBDIRECTORIES_ENABLE, true); + conf.setInt(DFSConfigKeys.FS_TRASH_INTERVAL_KEY, 3600); + MiniDFSCluster cluster = setupTestCase( + conf, testMatrixEntry.getProtectedPaths(), + testMatrixEntry.getUnprotectedPaths()); + + try { + LOG.info("Running {}", testMatrixEntry); + FileSystem fs = cluster.getFileSystem(); + for (Path srcPath : testMatrixEntry.getAllPathsToBeDeleted()) { + assertThat( + testMatrixEntry + ": Testing whether " + + srcPath + " can be moved to trash", + moveToTrash(fs, srcPath, conf), + is(testMatrixEntry.canPathBeRenamed(srcPath))); + } + } finally { + cluster.shutdown(); + } + } + } + @Test public void testDeleteProtectSubDirs() throws Throwable { for (TestMatrixEntry testMatrixEntry : @@ -465,6 +520,21 @@ private boolean deletePath(FileSystem fs, Path path) throws IOException { } } + private boolean moveToTrash(FileSystem fs, Path path, Configuration conf) { + try { + return Trash.moveToAppropriateTrash(fs, path, conf); + } catch (FileNotFoundException fnf) { + // fs.delete(...) does not throw an exception if the file does not exist. + // The deletePath method in this class, will therefore return true if + // there is an attempt to delete a file which does not exist. Therefore + // catching this exception and returning true to keep it consistent and + // allow tests to work with the same test matrix. + return true; + } catch (IOException ace) { + return false; + } + } + /** * Return true if the path was successfully renamed. False if it * failed with AccessControlException. Any other exceptions are From d8aaa8c3380451cd04bbc1febdd756a8db07d5c3 Mon Sep 17 00:00:00 2001 From: sguggilam Date: Wed, 26 Aug 2020 23:45:21 -0700 Subject: [PATCH 107/335] HADOOP-17159. Make UGI support forceful relogin from keytab ignoring the last login time (#2249) Contributed by Sandeep Guggilam. Signed-off-by: Mingliang Liu Signed-off-by: Steve Loughran --- .../hadoop/security/UserGroupInformation.java | 36 +++++++++++++++---- .../security/TestUGILoginFromKeytab.java | 36 +++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 91b64ade598e5..f0ddb786b6ff2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1232,7 +1232,29 @@ public void reloginFromKeytab() throws IOException { reloginFromKeytab(false); } + /** + * Force re-Login a user in from a keytab file irrespective of the last login + * time. Loads a user identity from a keytab file and logs them in. They + * become the currently logged-in user. This method assumes that + * {@link #loginUserFromKeytab(String, String)} had happened already. The + * Subject field of this UserGroupInformation object is updated to have the + * new credentials. + * + * @throws IOException + * @throws KerberosAuthException on a failure + */ + @InterfaceAudience.Public + @InterfaceStability.Evolving + public void forceReloginFromKeytab() throws IOException { + reloginFromKeytab(false, true); + } + private void reloginFromKeytab(boolean checkTGT) throws IOException { + reloginFromKeytab(checkTGT, false); + } + + private void reloginFromKeytab(boolean checkTGT, boolean ignoreLastLoginTime) + throws IOException { if (!shouldRelogin() || !isFromKeytab()) { return; } @@ -1247,7 +1269,7 @@ private void reloginFromKeytab(boolean checkTGT) throws IOException { return; } } - relogin(login); + relogin(login, ignoreLastLoginTime); } /** @@ -1268,25 +1290,27 @@ public void reloginFromTicketCache() throws IOException { if (login == null) { throw new KerberosAuthException(MUST_FIRST_LOGIN); } - relogin(login); + relogin(login, false); } - private void relogin(HadoopLoginContext login) throws IOException { + private void relogin(HadoopLoginContext login, boolean ignoreLastLoginTime) + throws IOException { // ensure the relogin is atomic to avoid leaving credentials in an // inconsistent state. prevents other ugi instances, SASL, and SPNEGO // from accessing or altering credentials during the relogin. synchronized(login.getSubjectLock()) { // another racing thread may have beat us to the relogin. if (login == getLogin()) { - unprotectedRelogin(login); + unprotectedRelogin(login, ignoreLastLoginTime); } } } - private void unprotectedRelogin(HadoopLoginContext login) throws IOException { + private void unprotectedRelogin(HadoopLoginContext login, + boolean ignoreLastLoginTime) throws IOException { assert Thread.holdsLock(login.getSubjectLock()); long now = Time.now(); - if (!hasSufficientTimeElapsed(now)) { + if (!hasSufficientTimeElapsed(now) && !ignoreLastLoginTime) { return; } // register most recent relogin attempt diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java index d233234c26c0c..db0095f2171e2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java @@ -158,6 +158,42 @@ public void testUGIReLoginFromKeytab() throws Exception { Assert.assertNotSame(login1, login2); } + /** + * Force re-login from keytab using the MiniKDC and verify the UGI can + * successfully relogin from keytab as well. + */ + @Test + public void testUGIForceReLoginFromKeytab() throws Exception { + // Set this to false as we are testing force re-login anyways + UserGroupInformation.setShouldRenewImmediatelyForTests(false); + String principal = "foo"; + File keytab = new File(workDir, "foo.keytab"); + kdc.createPrincipal(keytab, principal); + + UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + Assert.assertTrue("UGI should be configured to login from keytab", + ugi.isFromKeytab()); + + // Verify relogin from keytab. + User user = getUser(ugi.getSubject()); + final long firstLogin = user.getLastLogin(); + final LoginContext login1 = user.getLogin(); + Assert.assertNotNull(login1); + + // Sleep for 2 secs to have a difference between first and second login + Thread.sleep(2000); + + // Force relogin from keytab + ugi.forceReloginFromKeytab(); + final long secondLogin = user.getLastLogin(); + final LoginContext login2 = user.getLogin(); + Assert.assertTrue("User should have been able to relogin from keytab", + secondLogin > firstLogin); + Assert.assertNotNull(login2); + Assert.assertNotSame(login1, login2); + } + @Test public void testGetUGIFromKnownSubject() throws Exception { KerberosPrincipal principal = new KerberosPrincipal("user"); From 41182a9b6d81d0c8a4dc0a9cf89ea0ade815afd3 Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Thu, 27 Aug 2020 02:24:52 -0700 Subject: [PATCH 108/335] HDFS-15500. In-order deletion of snapshots: Diff lists must be update only in the last snapshot. (#2233) --- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 6 ++++++ .../namenode/snapshot/AbstractINodeDiffList.java | 6 +++++- .../namenode/snapshot/DiffListByArrayList.java | 2 ++ .../snapshot/DirectoryWithSnapshotFeature.java | 2 ++ .../server/namenode/snapshot/SnapshotManager.java | 12 ++++++++++++ 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 15bf6b16b569d..badf237049f7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1569,6 +1569,12 @@ public void checkOperation(OperationCategory op) throws StandbyException { // null in some unit tests haContext.checkOperation(op); } + + boolean assertsEnabled = false; + assert assertsEnabled = true; // Intentional side effect!!! + if (assertsEnabled && op == OperationCategory.WRITE) { + getSnapshotManager().initThreadLocals(); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java index 776adf1defc54..16e3b7536918e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java @@ -76,7 +76,11 @@ public final void deleteSnapshotDiff(INode.ReclaimContext reclaimContext, if (diffs == null) { return; } - int snapshotIndex = diffs.binarySearch(snapshot); + final int snapshotIndex = diffs.binarySearch(snapshot); + // DeletionOrdered: only can remove the element at index 0 and no prior + // check snapshotIndex <= 0 since the diff may not exist + assert !SnapshotManager.isDeletionOrdered() + || (snapshotIndex <= 0 && prior == Snapshot.NO_SNAPSHOT_ID); D removed; if (snapshotIndex == 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java index 95c23dfc02e27..7fa3f05efd0d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListByArrayList.java @@ -57,6 +57,8 @@ public int size() { @Override public T remove(int i) { + // DeletionOrdered: only can remove the element at index 0 + assert !SnapshotManager.isDeletionOrdered() || i == 0; return list.remove(i); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index b8f7b65ea7e05..c3a9aa14e9477 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -175,6 +175,8 @@ void combinePosteriorAndCollectBlocks( final INode.ReclaimContext reclaimContext, final INodeDirectory currentDir, final DirectoryDiff posterior) { + // DeletionOrdered: must not combine posterior + assert !SnapshotManager.isDeletionOrdered(); diff.combinePosterior(posterior.diff, new Diff.Processor() { /** Collect blocks for deleted files. */ @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index b5b0971298976..789fa3fa8b896 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -95,6 +95,18 @@ public class SnapshotManager implements SnapshotStatsMXBean { static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT = 5 * 60_000L; //5 minutes + private static final ThreadLocal DELETION_ORDERED + = new ThreadLocal<>(); + + static boolean isDeletionOrdered() { + final Boolean b = DELETION_ORDERED.get(); + return b != null? b: false; + } + + public void initThreadLocals() { + DELETION_ORDERED.set(isSnapshotDeletionOrdered()); + } + private final FSDirectory fsdir; private boolean captureOpenFiles; /** From d1c60a53f60a77c9004332d270955c41f85fe792 Mon Sep 17 00:00:00 2001 From: Mehakmeet Singh Date: Thu, 27 Aug 2020 15:57:00 +0530 Subject: [PATCH 109/335] HADOOP-17194. Adding Context class for AbfsClient in ABFS (#2216) Contributed by Mehakmeet Singh. --- .../fs/azurebfs/AzureBlobFileSystemStore.java | 41 +++++++++++-- .../fs/azurebfs/services/AbfsClient.java | 24 +++----- .../azurebfs/services/AbfsClientContext.java | 51 ++++++++++++++++ .../services/AbfsClientContextBuilder.java | 58 +++++++++++++++++++ .../fs/azurebfs/services/TestAbfsClient.java | 12 +++- 5 files changed, 161 insertions(+), 25 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContext.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContextBuilder.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 59c2e263b25b9..9861e3a772103 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -84,6 +84,8 @@ import org.apache.hadoop.fs.azurebfs.oauth2.IdentityTransformerInterface; import org.apache.hadoop.fs.azurebfs.services.AbfsAclHelper; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientContext; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientContextBuilder; import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream; @@ -146,6 +148,7 @@ public class AzureBlobFileSystemStore implements Closeable { private final UserGroupInformation userGroupInformation; private final IdentityTransformerInterface identityTransformer; private final AbfsPerfTracker abfsPerfTracker; + private final AbfsCounters abfsCounters; /** * The set of directories where we should store files as append blobs. @@ -192,7 +195,8 @@ public AzureBlobFileSystemStore(URI uri, boolean isSecureScheme, boolean usingOauth = (authType == AuthType.OAuth); boolean useHttps = (usingOauth || abfsConfiguration.isHttpsAlwaysUsed()) ? true : isSecureScheme; this.abfsPerfTracker = new AbfsPerfTracker(fileSystemName, accountName, this.abfsConfiguration); - initializeClient(uri, fileSystemName, accountName, useHttps, abfsCounters); + this.abfsCounters = abfsCounters; + initializeClient(uri, fileSystemName, accountName, useHttps); final Class identityTransformerClass = configuration.getClass(FS_AZURE_IDENTITY_TRANSFORM_CLASS, IdentityTransformer.class, IdentityTransformerInterface.class); @@ -1213,8 +1217,19 @@ public boolean isAtomicRenameKey(String key) { return isKeyForDirectorySet(key, azureAtomicRenameDirSet); } + /** + * A on-off operation to initialize AbfsClient for AzureBlobFileSystem + * Operations. + * + * @param uri Uniform resource identifier for Abfs. + * @param fileSystemName Name of the fileSystem being used. + * @param accountName Name of the account being used to access Azure + * data store. + * @param isSecure Tells if https is being used or http. + * @throws IOException + */ private void initializeClient(URI uri, String fileSystemName, - String accountName, boolean isSecure, AbfsCounters abfsCounters) + String accountName, boolean isSecure) throws IOException { if (this.client != null) { return; @@ -1261,16 +1276,30 @@ private void initializeClient(URI uri, String fileSystemName, LOG.trace("Initializing AbfsClient for {}", baseUrl); if (tokenProvider != null) { this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, - new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries()), - tokenProvider, abfsPerfTracker, abfsCounters); + tokenProvider, + populateAbfsClientContext()); } else { this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, - new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries()), - sasTokenProvider, abfsPerfTracker, abfsCounters); + sasTokenProvider, + populateAbfsClientContext()); } LOG.trace("AbfsClient init complete"); } + /** + * Populate a new AbfsClientContext instance with the desired properties. + * + * @return an instance of AbfsClientContext. + */ + private AbfsClientContext populateAbfsClientContext() { + return new AbfsClientContextBuilder() + .withExponentialRetryPolicy( + new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries())) + .withAbfsCounters(abfsCounters) + .withAbfsPerfTracker(abfsPerfTracker) + .build(); + } + private String getOctalNotation(FsPermission fsPermission) { Preconditions.checkNotNull(fsPermission, "fsPermission"); return String.format(AbfsHttpConstants.PERMISSION_FORMAT, fsPermission.toOctal()); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 3ee2d7e8e19d3..45c1948a0ec7a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -77,15 +77,13 @@ public class AbfsClient implements Closeable { private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials, final AbfsConfiguration abfsConfiguration, - final ExponentialRetryPolicy exponentialRetryPolicy, - final AbfsPerfTracker abfsPerfTracker, - final AbfsCounters abfsCounters) { + final AbfsClientContext abfsClientContext) { this.baseUrl = baseUrl; this.sharedKeyCredentials = sharedKeyCredentials; String baseUrlString = baseUrl.toString(); this.filesystem = baseUrlString.substring(baseUrlString.lastIndexOf(FORWARD_SLASH) + 1); this.abfsConfiguration = abfsConfiguration; - this.retryPolicy = exponentialRetryPolicy; + this.retryPolicy = abfsClientContext.getExponentialRetryPolicy(); this.accountName = abfsConfiguration.getAccountName().substring(0, abfsConfiguration.getAccountName().indexOf(AbfsHttpConstants.DOT)); this.authType = abfsConfiguration.getAuthType(accountName); @@ -105,29 +103,23 @@ private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCreden } this.userAgent = initializeUserAgent(abfsConfiguration, sslProviderName); - this.abfsPerfTracker = abfsPerfTracker; - this.abfsCounters = abfsCounters; + this.abfsPerfTracker = abfsClientContext.getAbfsPerfTracker(); + this.abfsCounters = abfsClientContext.getAbfsCounters(); } public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials, final AbfsConfiguration abfsConfiguration, - final ExponentialRetryPolicy exponentialRetryPolicy, final AccessTokenProvider tokenProvider, - final AbfsPerfTracker abfsPerfTracker, - final AbfsCounters abfsCounters) { - this(baseUrl, sharedKeyCredentials, abfsConfiguration, - exponentialRetryPolicy, abfsPerfTracker, abfsCounters); + final AbfsClientContext abfsClientContext) { + this(baseUrl, sharedKeyCredentials, abfsConfiguration, abfsClientContext); this.tokenProvider = tokenProvider; } public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials, final AbfsConfiguration abfsConfiguration, - final ExponentialRetryPolicy exponentialRetryPolicy, final SASTokenProvider sasTokenProvider, - final AbfsPerfTracker abfsPerfTracker, - final AbfsCounters abfsCounters) { - this(baseUrl, sharedKeyCredentials, abfsConfiguration, - exponentialRetryPolicy, abfsPerfTracker, abfsCounters); + final AbfsClientContext abfsClientContext) { + this(baseUrl, sharedKeyCredentials, abfsConfiguration, abfsClientContext); this.sasTokenProvider = sasTokenProvider; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContext.java new file mode 100644 index 0000000000000..ad20550af7c3f --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContext.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +/** + * Class to hold extra configurations for AbfsClient and further classes + * inside AbfsClient. + */ +public class AbfsClientContext { + + private final ExponentialRetryPolicy exponentialRetryPolicy; + private final AbfsPerfTracker abfsPerfTracker; + private final AbfsCounters abfsCounters; + + AbfsClientContext( + ExponentialRetryPolicy exponentialRetryPolicy, + AbfsPerfTracker abfsPerfTracker, + AbfsCounters abfsCounters) { + this.exponentialRetryPolicy = exponentialRetryPolicy; + this.abfsPerfTracker = abfsPerfTracker; + this.abfsCounters = abfsCounters; + } + + public ExponentialRetryPolicy getExponentialRetryPolicy() { + return exponentialRetryPolicy; + } + + public AbfsPerfTracker getAbfsPerfTracker() { + return abfsPerfTracker; + } + + public AbfsCounters getAbfsCounters() { + return abfsCounters; + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContextBuilder.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContextBuilder.java new file mode 100644 index 0000000000000..00513f7138d53 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientContextBuilder.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +/** + * A builder for AbfsClientContext class with different options to select and + * build from. + */ +public class AbfsClientContextBuilder { + + private ExponentialRetryPolicy exponentialRetryPolicy; + private AbfsPerfTracker abfsPerfTracker; + private AbfsCounters abfsCounters; + + public AbfsClientContextBuilder withExponentialRetryPolicy( + final ExponentialRetryPolicy exponentialRetryPolicy) { + this.exponentialRetryPolicy = exponentialRetryPolicy; + return this; + } + + public AbfsClientContextBuilder withAbfsPerfTracker( + final AbfsPerfTracker abfsPerfTracker) { + this.abfsPerfTracker = abfsPerfTracker; + return this; + } + + public AbfsClientContextBuilder withAbfsCounters(final AbfsCounters abfsCounters) { + this.abfsCounters = abfsCounters; + return this; + } + + /** + * Build the context and get the instance with the properties selected. + * + * @return an instance of AbfsClientContext. + */ + public AbfsClientContext build() { + //validate the values + return new AbfsClientContext(exponentialRetryPolicy, abfsPerfTracker, + abfsCounters); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index bab02c09c7423..0d904c85523e7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -103,8 +103,9 @@ public TestAbfsClient(){ private String getUserAgentString(AbfsConfiguration config, boolean includeSSLProvider) throws MalformedURLException { + AbfsClientContext abfsClientContext = new AbfsClientContextBuilder().build(); AbfsClient client = new AbfsClient(new URL("https://azure.com"), null, - config, null, (AccessTokenProvider) null, null, null); + config, (AccessTokenProvider) null, abfsClientContext); String sslProviderName = null; if (includeSSLProvider) { sslProviderName = DelegatingSSLSocketFactory.getDefaultFactory() @@ -257,6 +258,12 @@ public static AbfsClient createTestClientFromCurrentContext( abfsConfig.getAccountName(), abfsConfig); + AbfsClientContext abfsClientContext = + new AbfsClientContextBuilder().withAbfsPerfTracker(tracker) + .withExponentialRetryPolicy( + new ExponentialRetryPolicy(abfsConfig.getMaxIoRetries())) + .build(); + // Create test AbfsClient AbfsClient testClient = new AbfsClient( baseAbfsClientInstance.getBaseUrl(), @@ -267,11 +274,10 @@ public static AbfsClient createTestClientFromCurrentContext( abfsConfig.getStorageAccountKey()) : null), abfsConfig, - new ExponentialRetryPolicy(abfsConfig.getMaxIoRetries()), (currentAuthType == AuthType.OAuth ? abfsConfig.getTokenProvider() : null), - tracker, null); + abfsClientContext); return testClient; } From 06793da100123bda67afcda95d6f2221efd9c281 Mon Sep 17 00:00:00 2001 From: Vivek Ratnavel Subramanian Date: Thu, 27 Aug 2020 11:36:30 -0700 Subject: [PATCH 110/335] HDFS-15531. Namenode UI: List snapshots in separate table for each snapshottable directory (#2230) --- .../src/main/webapps/hdfs/dfshealth.html | 53 +++----- .../src/main/webapps/hdfs/dfshealth.js | 117 ++++++++++++++++++ .../src/main/webapps/static/hadoop.css | 33 ++++- 3 files changed, 167 insertions(+), 36 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 8b03185d3d1d6..27616fb6b9b6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -250,13 +250,15 @@ - + diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index e1ea5840deda5..911881347e933 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -416,7 +416,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> src/main/webapps/static/moment.min.js src/main/webapps/static/dust-full-2.0.0.min.js src/main/webapps/static/dust-helpers-1.1.1.min.js - src/main/webapps/static/jquery-3.4.1.min.js + src/main/webapps/static/jquery-3.5.1.min.js src/main/webapps/static/jquery.dataTables.min.js src/main/webapps/static/json-bignum.js src/main/webapps/static/dataTables.bootstrap.css diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html index 39680e84a8c82..7301064651e2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html @@ -125,7 +125,7 @@ {/dn.VolumeInfo} - + @@ -134,4 +134,4 @@ - \ No newline at end of file + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 27616fb6b9b6f..bb3cf4a475ca8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -476,7 +476,7 @@ - - - + @@ -105,4 +105,4 @@ - \ No newline at end of file + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html index e9c206eda0381..41a468d4cfd80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html @@ -86,7 +86,7 @@ {/snn} -