From 3a89471c36678d3f663528ad599d59d8b63ae9cd Mon Sep 17 00:00:00 2001
From: He Xiaoqiao <hexiaoqiao@apache.org>
Date: Wed, 7 Apr 2021 12:18:23 +0800
Subject: [PATCH 01/43] HADOOP-17613. Log not flushed fully when daemon
 shutdown. Contributed by Renukaprasad C.

---
 .../src/main/java/org/apache/hadoop/util/StringUtils.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 9e7b36f71e211..4fe09a9ed7e6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.log4j.LogManager;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
@@ -752,6 +753,7 @@ static void startupShutdownMessage(Class<?> clazz, String[] args,
         public void run() {
           LOG.info(toStartupShutdownString("SHUTDOWN_MSG: ", new String[]{
             "Shutting down " + classname + " at " + hostname}));
+          LogManager.shutdown();
         }
       }, SHUTDOWN_HOOK_PRIORITY);
 

From f608cda72976d62354e063472d9b985eed48ebae Mon Sep 17 00:00:00 2001
From: Wei-Chiu Chuang <weichiu@apache.org>
Date: Tue, 6 Apr 2021 21:52:23 -0700
Subject: [PATCH 02/43] HADOOP-17624. Remove any rocksdb exclusion code.
 (#2867)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reviewed-by: Márton Elek <elek@apache.org>
Reviewed-by: Viraj Jasani <vjasani@apache.org>
---
 .../test/resources/ensure-jars-have-correct-contents.sh    | 7 -------
 hadoop-client-modules/hadoop-client-minicluster/pom.xml    | 7 -------
 2 files changed, 14 deletions(-)

diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
index d77424e6b7899..0dbfefbf4f16d 100644
--- a/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
+++ b/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
@@ -58,13 +58,6 @@ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
 allowed_expr+="|^java.policy$"
 #   * Used by javax.annotation
 allowed_expr+="|^jndi.properties$"
-# * allowing native libraries from rocksdb. Leaving native libraries as it is.
-allowed_expr+="|^librocksdbjni-linux32.so"
-allowed_expr+="|^librocksdbjni-linux64.so"
-allowed_expr+="|^librocksdbjni-osx.jnilib"
-allowed_expr+="|^librocksdbjni-win64.dll"
-allowed_expr+="|^librocksdbjni-linux-ppc64le.so"
-
 
 allowed_expr+=")"
 declare -i bad_artifacts=0
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index 48dc65dc7cb04..a35d832a76c04 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -760,13 +760,6 @@
                         <exclude>xml.xsd</exclude>
                       </excludes>
                     </filter>
-                    <!-- filtering HISTORY-JAVA.md from rocksdb jar -->
-                    <filter>
-                    <artifact>org.rocksdb:rocksdbjni</artifact>
-                    <excludes>
-                      <exclude>HISTORY-JAVA.md</exclude>
-                    </excludes>
-                    </filter>
                     <filter>
                       <!-- skip jetty license info already incorporated into LICENSE/NOTICE -->
                       <artifact>org.eclipse.jetty:*</artifact>

From dac60b8282013d7776667415a429e7ca35efba66 Mon Sep 17 00:00:00 2001
From: Wei-Chiu Chuang <weichiu@apache.org>
Date: Tue, 6 Apr 2021 21:55:01 -0700
Subject: [PATCH 03/43] HADOOP-17621. hadoop-auth to remove jetty-server
 dependency. (#2865)

Reviewed-by: Akira Ajisaka <aajisaka@apache.org>
---
 hadoop-common-project/hadoop-auth/pom.xml                 | 6 +-----
 .../authentication/server/AuthenticationFilter.java       | 8 ++------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 8e8526c7450e0..89d54198feeec 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -193,11 +193,7 @@
       <artifactId>guava</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-server</artifactId>
-    </dependency>
-  </dependencies>
+ </dependencies>
 
   <build>
     <plugins>
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 9f40c42d24135..b339a5d5a2450 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -19,7 +19,6 @@
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
 import org.apache.hadoop.security.authentication.util.*;
-import org.eclipse.jetty.server.Response;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -621,7 +620,7 @@ && getMaxInactiveInterval() > 0) {
           errCode = HttpServletResponse.SC_FORBIDDEN;
         }
         // After Jetty 9.4.21, sendError() no longer allows a custom message.
-        // use setStatusWithReason() to set a custom message.
+        // use setStatus() to set a custom message.
         String reason;
         if (authenticationEx == null) {
           reason = "Authentication required";
@@ -629,10 +628,7 @@ && getMaxInactiveInterval() > 0) {
           reason = authenticationEx.getMessage();
         }
 
-        if (httpResponse instanceof Response) {
-          ((Response)httpResponse).setStatusWithReason(errCode, reason);
-        }
-
+        httpResponse.setStatus(errCode, reason);
         httpResponse.sendError(errCode, reason);
       }
     }

From 9cd69c20c4aaad51ce77933ed430b66266156ca7 Mon Sep 17 00:00:00 2001
From: Szilard Nemeth <snemeth@apache.org>
Date: Wed, 7 Apr 2021 11:52:21 +0200
Subject: [PATCH 04/43] YARN-10714. Remove dangling dynamic queues on
 reinitialization. Contributed by Andras Gyori

---
 .../CapacitySchedulerQueueManager.java        | 43 ++++++++++---
 ...CapacitySchedulerNewQueueAutoCreation.java | 61 +++++++++++++++++++
 2 files changed, 97 insertions(+), 7 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 00d1cda30886e..5cd14908e8914 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -27,8 +27,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
-import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -340,11 +338,14 @@ private void updateQueues(CSQueueStore existingQueues,
     }
 
     for (CSQueue queue : existingQueues.getQueues()) {
-      if (!((AbstractCSQueue) queue).isDynamicQueue() && newQueues.get(
-          queue.getQueuePath()) == null && !(
-          queue instanceof AutoCreatedLeafQueue && conf
-              .isAutoCreateChildQueueEnabled(
-                  queue.getParent().getQueuePath()))) {
+      boolean isDanglingDynamicQueue = isDanglingDynamicQueue(
+          newQueues, existingQueues, queue);
+      boolean isRemovable = isDanglingDynamicQueue || !isDynamicQueue(queue)
+          && newQueues.get(queue.getQueuePath()) == null
+          && !(queue instanceof AutoCreatedLeafQueue &&
+          conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePath()));
+
+      if (isRemovable) {
         existingQueues.remove(queue);
       }
     }
@@ -435,4 +436,32 @@ private Map<String, Set<String>> getQueueToLabels() {
       getQueueStateManager() {
     return this.queueStateManager;
   }
+
+  private boolean isDynamicQueue(CSQueue queue) {
+    return (queue instanceof AbstractCSQueue) &&
+        ((AbstractCSQueue) queue).isDynamicQueue();
+  }
+
+  private boolean isDanglingDynamicQueue(
+      CSQueueStore newQueues, CSQueueStore existingQueues,
+      CSQueue queue) {
+    if (!isDynamicQueue(queue)) {
+      return false;
+    }
+    if (queue.getParent() == null) {
+      return true;
+    }
+    if (newQueues.get(queue.getParent().getQueuePath()) != null) {
+      return false;
+    }
+    CSQueue parent = existingQueues.get(queue.getParent().getQueuePath());
+    if (parent == null) {
+      return true;
+    }
+    // A dynamic queue is dangling, if its parent is not parsed in newQueues
+    // or if its parent is not a dynamic queue. Dynamic queues are not parsed in
+    // newQueues but they are deleted automatically, so it is safe to assume
+    // that existingQueues contain valid dynamic queues.
+    return !isDynamicQueue(parent);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index c403d23f07b98..4dae4fd64eb3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -968,6 +969,66 @@ public void testQueueInfoIfAmbiguousQueueNames() throws Exception {
         bAutoLeafQueue.getQueueInfo().getQueueName());
   }
 
+  @Test
+  public void testRemoveDanglingAutoCreatedQueuesOnReinit() throws Exception {
+    startScheduler();
+
+    // Validate static parent deletion
+    createQueue("root.a.a-auto");
+    AbstractCSQueue aAuto = (AbstractCSQueue) cs.
+        getQueue("root.a.a-auto");
+    Assert.assertTrue(aAuto.isDynamicQueue());
+
+    csConf.setState("root.a", QueueState.STOPPED);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    aAuto = (AbstractCSQueue) cs.
+        getQueue("root.a.a-auto");
+    Assert.assertEquals("root.a.a-auto is not in STOPPED state", QueueState.STOPPED, aAuto.getState());
+    csConf.setQueues("root", new String[]{"b"});
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    CSQueue aAutoNew = cs.getQueue("root.a.a-auto");
+    Assert.assertNull(aAutoNew);
+
+    submitApp(cs, USER0, "a-auto", "root.a");
+    aAutoNew = cs.getQueue("root.a.a-auto");
+    Assert.assertNotNull(aAutoNew);
+
+    // Validate static grandparent deletion
+    csConf.setQueues("root", new String[]{"a", "b"});
+    csConf.setQueues("root.a", new String[]{"a1"});
+    csConf.setAutoQueueCreationV2Enabled("root.a.a1", true);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    createQueue("root.a.a1.a1-auto");
+    CSQueue a1Auto = cs.getQueue("root.a.a1.a1-auto");
+    Assert.assertNotNull("a1-auto should exist", a1Auto);
+
+    csConf.setQueues("root", new String[]{"b"});
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    a1Auto = cs.getQueue("root.a.a1.a1-auto");
+    Assert.assertNull("a1-auto has no parent and should not exist", a1Auto);
+
+    // Validate dynamic parent deletion
+    csConf.setState("root.b", QueueState.STOPPED);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    csConf.setAutoQueueCreationV2Enabled("root.b", true);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    createQueue("root.b.b-auto-parent.b-auto-leaf");
+    CSQueue bAutoParent = cs.getQueue("root.b.b-auto-parent");
+    Assert.assertNotNull("b-auto-parent should exist", bAutoParent);
+    ParentQueue b = (ParentQueue) cs.getQueue("root.b");
+    b.removeChildQueue(bAutoParent);
+
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    bAutoParent = cs.getQueue("root.b.b-auto-parent");
+    Assert.assertNull("b-auto-parent should not exist ", bAutoParent);
+    CSQueue bAutoLeaf = cs.getQueue("root.b.b-auto-parent.b-auto-leaf");
+    Assert.assertNull("b-auto-leaf should not exist " +
+        "when its dynamic parent is removed", bAutoLeaf);
+  }
+
   protected LeafQueue createQueue(String queuePath) throws YarnException {
     return autoQueueHandler.autoCreateQueue(
         CSQueueUtils.extractQueuePath(queuePath));

From 02b9506c5a7be214c185fe1fb8aad214b3210bb8 Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Wed, 7 Apr 2021 15:36:25 +0530
Subject: [PATCH 05/43] HDFS-15948. Fix test4tests for libhdfspp (#2873)

Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
---
 dev-support/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index d2266c5530bc2..f5305429264cd 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -35,7 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='11eb9b09786e401fbdeaa3be83a19a4066fd7813'
+        YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
     }
 
     parameters {

From ae88174c29ae02b6cf48785ecb3432a2698944bb Mon Sep 17 00:00:00 2001
From: Brahma Reddy Battula <brahma@apache.org>
Date: Wed, 7 Apr 2021 23:49:17 +0530
Subject: [PATCH 06/43] HADOOP-17617. Incorrect representation of RESPONSE for
 Get Key Version in KMS index.md.vm file. Contributed by  Ravuri Sushma sree

---
 .../hadoop-kms/src/site/markdown/index.md.vm             | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 95e926b3561a1..d7599de556900 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -1055,7 +1055,8 @@ $H4 Get Key Version
     Content-Type: application/json
 
     {
-      "name"        : "versionName",
+      "name"        : "<name>",
+      "versionName" : "<version>",
       "material"    : "<material>",    //base64
     }
 
@@ -1072,11 +1073,13 @@ $H4 Get Key Versions
 
     [
       {
-        "name"        : "versionName",
+        "name"        : "<name>",
+        "versionName" : "<version>",
         "material"    : "<material>",    //base64
       },
       {
-        "name"        : "versionName",
+        "name"        : "<name>",
+        "versionName" : "<version>",
         "material"    : "<material>",    //base64
       },
       ...

From 6040e86e99aae5e29c17b03fddb0a805da8fcae8 Mon Sep 17 00:00:00 2001
From: Wei-Chiu Chuang <weichiu@apache.org>
Date: Wed, 7 Apr 2021 23:25:11 -0700
Subject: [PATCH 07/43] HADOOP-17625. Update to Jetty 9.4.39. (#2870)

Reviewed-by: cxorm <lianp964@gmail.com>
---
 hadoop-project/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index bb021fcf2c0f8..5f81d75605a3d 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -37,7 +37,7 @@
     <!--Whether to proceed to next module if any test failures exist-->
     <ignoreTestFailure>true</ignoreTestFailure>
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
-    <jetty.version>9.4.35.v20201120</jetty.version>
+    <jetty.version>9.4.39.v20210325</jetty.version>
     <test.exclude>_</test.exclude>
     <test.exclude.pattern>_</test.exclude.pattern>
 

From 46a59798055befb60ba62ff254a4d5ac300f5b0d Mon Sep 17 00:00:00 2001
From: Sungpeo Kook <justkook@gmail.com>
Date: Thu, 8 Apr 2021 15:58:10 +0900
Subject: [PATCH 08/43] MAPREDUCE-7270. TestHistoryViewerPrinter could be
 failed when the locale isn't English. (#1942)

Contributed by Sungpeo Kook.
Signed-off-by: Mingliang Liu <liuml07@apache.org>
---
 .../jobhistory/TestHistoryViewerPrinter.java      | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
index f0f713a98211d..cb508f6d29b9d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
@@ -24,7 +24,9 @@
 import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.JobStatus;
 import org.apache.hadoop.mapreduce.TaskType;
+import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.skyscreamer.jsonassert.JSONAssert;
 import org.skyscreamer.jsonassert.JSONCompareMode;
@@ -35,6 +37,7 @@
 import java.io.PrintStream;
 import java.util.HashMap;
 import java.util.TimeZone;
+import java.util.Locale;
 
 public class TestHistoryViewerPrinter {
 
@@ -43,6 +46,18 @@ public class TestHistoryViewerPrinter {
 
   private final String LINE_SEPARATOR = System.lineSeparator();
 
+  private static final Locale DEFAULT_LOCALE = Locale.getDefault();
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    Locale.setDefault(Locale.ENGLISH);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    Locale.setDefault(DEFAULT_LOCALE);
+  }
+
   @Test
   public void testHumanPrinter() throws Exception {
     JobHistoryParser.JobInfo job = createJobInfo();

From ca9aa91d102a388c6965ac0bf91e99ba8ef536b4 Mon Sep 17 00:00:00 2001
From: Peter Bacsko <pbacsko@cloudera.com>
Date: Thu, 8 Apr 2021 12:42:48 +0200
Subject: [PATCH 09/43] YARN-10564. Support Auto Queue Creation template
 configurations. Contributed by Andras Gyori.

---
 .../scheduler/capacity/AbstractCSQueue.java   |  35 ++---
 .../capacity/AutoCreatedQueueTemplate.java    | 133 ++++++++++++++++++
 .../CapacitySchedulerConfiguration.java       |   2 +-
 .../scheduler/capacity/ParentQueue.java       |  27 +++-
 .../TestAutoCreatedQueueTemplate.java         | 116 +++++++++++++++
 ...CapacitySchedulerNewQueueAutoCreation.java |  80 +++++++++++
 6 files changed, 375 insertions(+), 18 deletions(-)
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
 create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 06575be4c7a3e..250fcc716d645 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -18,25 +18,15 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -58,13 +48,13 @@
 import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerAllocationProposal;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
@@ -74,8 +64,17 @@
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED;
 
@@ -361,6 +360,10 @@ protected void setupQueueConfigs(Resource clusterResource,
 
     writeLock.lock();
     try {
+      if (isDynamicQueue() && getParent() instanceof ParentQueue) {
+        ((ParentQueue) getParent()).getAutoCreatedQueueTemplate()
+            .setTemplateEntriesForChild(configuration, getQueuePath());
+      }
       // get labels
       this.accessibleLabels =
           configuration.getAccessibleNodeLabels(getQueuePath());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
new file mode 100644
index 0000000000000..6c516c04770b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX;
+
+/**
+ * A handler for storing and setting auto created queue template settings.
+ */
+public class AutoCreatedQueueTemplate {
+  public static final String AUTO_QUEUE_TEMPLATE_PREFIX =
+      AUTO_QUEUE_CREATION_V2_PREFIX + "template.";
+  private static final String WILDCARD_QUEUE = "*";
+  private static final int MAX_WILDCARD_LEVEL = 1;
+
+  private final Map<String, String> templateProperties = new HashMap<>();
+
+  public AutoCreatedQueueTemplate(Configuration configuration,
+                                  String queuePath) {
+    setTemplateConfigEntries(configuration, queuePath);
+  }
+
+  @VisibleForTesting
+  public static String getAutoQueueTemplatePrefix(String queue) {
+    return CapacitySchedulerConfiguration.getQueuePrefix(queue)
+        + AUTO_QUEUE_TEMPLATE_PREFIX;
+  }
+
+  /**
+   * Get the template properties attached to a parent queue.
+   * @return template property names and values
+   */
+  public Map<String, String> getTemplateProperties() {
+    return templateProperties;
+  }
+
+  /**
+   * Sets the configuration properties of a child queue based on its parent
+   * template settings.
+   * @param conf configuration to set
+   * @param childQueuePath child queue path used for prefixing the properties
+   */
+  public void setTemplateEntriesForChild(Configuration conf,
+                                         String childQueuePath) {
+    // Get all properties that are explicitly set
+    Set<String> alreadySetProps = conf.getPropsWithPrefix(
+        CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet();
+
+    for (Map.Entry<String, String> entry : templateProperties.entrySet()) {
+      // Do not overwrite explicitly configured properties
+      if (alreadySetProps.contains(entry.getKey())) {
+        continue;
+      }
+      conf.set(CapacitySchedulerConfiguration.getQueuePrefix(
+          childQueuePath) + entry.getKey(), entry.getValue());
+    }
+  }
+
+  /**
+   * Store the template configuration properties. Explicit templates always take
+   * precedence over wildcard values. An example template precedence
+   * hierarchy for root.a ParentQueue from highest to lowest:
+   * yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity
+   * yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity
+   */
+  private void setTemplateConfigEntries(Configuration configuration,
+                                        String queuePath) {
+    List<String> queuePathParts = new ArrayList<>(Arrays.asList(
+        queuePath.split("\\.")));
+
+    if (queuePathParts.size() <= 1) {
+      // This is either root or an empty queue name
+      return;
+    }
+    int queuePathMaxIndex = queuePathParts.size() - 1;
+
+    // start with the most explicit format (without wildcard)
+    int wildcardLevel = 0;
+    // root can not be wildcarded
+    // MAX_WILDCARD_LEVEL will be configurable in the future
+    int supportedWildcardLevel = Math.min(queuePathMaxIndex - 1,
+        MAX_WILDCARD_LEVEL);
+
+
+    // Collect all template entries
+    while (wildcardLevel <= supportedWildcardLevel) {
+      // Get all config entries with the specified prefix
+      String templateQueuePath = String.join(".", queuePathParts);
+      // Get all configuration entries with
+      // <queuePath>.auto-queue-creation-v2.template prefix
+      Map<String, String> props = configuration.getPropsWithPrefix(
+          getAutoQueueTemplatePrefix(templateQueuePath));
+
+      for (Map.Entry<String, String> entry : props.entrySet()) {
+        // If an entry is already present, it had a higher precedence
+        templateProperties.putIfAbsent(entry.getKey(), entry.getValue());
+      }
+
+      // Replace a queue part with a wildcard based on the wildcard level
+      // eg. root.a -> root.*
+      int queuePartToWildcard = queuePathMaxIndex - wildcardLevel;
+      queuePathParts.set(queuePartToWildcard, WILDCARD_QUEUE);
+
+      ++wildcardLevel;
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 5bae2b375f1a8..266cbb4d34601 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -2029,7 +2029,7 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
       AUTO_CREATE_CHILD_QUEUE_PREFIX + "enabled";
 
   @Private
-  private static final String AUTO_QUEUE_CREATION_V2_PREFIX =
+  protected static final String AUTO_QUEUE_CREATION_V2_PREFIX =
       "auto-queue-creation-v2.";
 
   @Private
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 3d28933141359..798c71037845d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.slf4j.Logger;
@@ -101,6 +102,8 @@ public class ParentQueue extends AbstractCSQueue {
 
   private final boolean allowZeroCapacitySum;
 
+  private AutoCreatedQueueTemplate autoCreatedQueueTemplate;
+
   // effective min ratio per resource, it is used during updateClusterResource,
   // leaf queue can use this to calculate effective resources.
   // This field will not be edited, reference will point to a new immutable map
@@ -152,6 +155,8 @@ protected void setupQueueConfigs(Resource clusterResource,
       throws IOException {
     writeLock.lock();
     try {
+      autoCreatedQueueTemplate = new AutoCreatedQueueTemplate(
+          csConf, getQueuePath());
       super.setupQueueConfigs(clusterResource, csConf);
       StringBuilder aclsString = new StringBuilder();
       for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
@@ -477,6 +482,8 @@ private CapacitySchedulerConfiguration getConfForAutoCreatedQueue(
     CapacitySchedulerConfiguration dupCSConfig =
         new CapacitySchedulerConfiguration(
         csContext.getConfiguration(), false);
+    autoCreatedQueueTemplate.setTemplateEntriesForChild(dupCSConfig,
+        childQueuePath);
     if (isLeaf) {
       // set to -1, to disable it
       dupCSConfig.setUserLimitFactor(childQueuePath, -1);
@@ -647,6 +654,18 @@ public void reinitialize(CSQueue newlyParsedQueue,
       Map<String, CSQueue> currentChildQueues = getQueuesMap(childQueues);
       Map<String, CSQueue> newChildQueues = getQueuesMap(
           newlyParsedParentQueue.childQueues);
+
+      // Reinitialize dynamic queues as well, because they are not parsed
+      for (String queue : Sets.difference(currentChildQueues.keySet(),
+          newChildQueues.keySet())) {
+        CSQueue candidate = currentChildQueues.get(queue);
+        if (candidate instanceof AbstractCSQueue) {
+          if (((AbstractCSQueue) candidate).isDynamicQueue()) {
+            candidate.reinitialize(candidate, clusterResource);
+          }
+        }
+      }
+
       for (Map.Entry<String, CSQueue> e : newChildQueues.entrySet()) {
         String newChildQueueName = e.getKey();
         CSQueue newChildQueue = e.getValue();
@@ -1217,7 +1236,9 @@ public void updateClusterResource(Resource clusterResource,
           // For dynamic queue, we will set weight to 1 every time, because it
           // is possible new labels added to the parent.
           if (((AbstractCSQueue) queue).isDynamicQueue()) {
-            queue.getQueueCapacities().setWeight(nodeLabel, 1f);
+            if (queue.getQueueCapacities().getWeight(nodeLabel) == -1f) {
+              queue.getQueueCapacities().setWeight(nodeLabel, 1f);
+            }
           }
         }
       }
@@ -1637,4 +1658,8 @@ public boolean isEligibleForAutoDeletion() {
         csContext.getConfiguration().
             isAutoExpiredDeletionEnabled(this.getQueuePath());
   }
+
+  public AutoCreatedQueueTemplate getAutoCreatedQueueTemplate() {
+    return autoCreatedQueueTemplate;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
new file mode 100644
index 0000000000000..1c021f6efb04d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAutoCreatedQueueTemplate {
+  private static final String TEST_QUEUE_ABC = "root.a.b.c";
+  private static final String TEST_QUEUE_AB = "root.a.b";
+  private static final String TEST_QUEUE_A = "root.a";
+  private static final String ROOT = "root";
+  private CapacitySchedulerConfiguration conf;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new CapacitySchedulerConfiguration();
+    conf.setQueues("root", new String[]{"a"});
+    conf.setQueues("a", new String[]{"b"});
+    conf.setQueues("b", new String[]{"c"});
+
+  }
+
+  @Test
+  public void testNonWildCardTemplate() {
+    conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w");
+    AutoCreatedQueueTemplate template =
+        new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+
+    Assert.assertEquals("weight is not set", 6f,
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+
+  }
+
+  @Test
+  public void testOneLevelWildcardTemplate() {
+    conf.set(getTemplateKey("root.a.*", "capacity"), "6w");
+    AutoCreatedQueueTemplate template =
+        new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+
+    Assert.assertEquals("weight is not set", 6f,
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+
+  }
+
+  @Test
+  public void testIgnoredWhenRootWildcarded() {
+    conf.set(getTemplateKey("*", "capacity"), "6w");
+    AutoCreatedQueueTemplate template =
+        new AutoCreatedQueueTemplate(conf, ROOT);
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
+
+    Assert.assertEquals("weight is set", -1f,
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
+  }
+
+  @Test
+  public void testIgnoredWhenNoParent() {
+    conf.set(getTemplateKey("root", "capacity"), "6w");
+    AutoCreatedQueueTemplate template =
+        new AutoCreatedQueueTemplate(conf, ROOT);
+    template.setTemplateEntriesForChild(conf, ROOT);
+
+    Assert.assertEquals("weight is set", -1f,
+        conf.getNonLabeledQueueWeight(ROOT), 10e-6);
+  }
+
+  @Test
+  public void testTemplatePrecedence() {
+    conf.set(getTemplateKey("root.a.b", "capacity"), "6w");
+    conf.set(getTemplateKey("root.a.*", "capacity"), "4w");
+    conf.set(getTemplateKey("root.*.*", "capacity"), "2w");
+
+    AutoCreatedQueueTemplate template =
+        new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+
+    Assert.assertEquals(
+        "explicit template does not have the highest precedence", 6f,
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+
+    CapacitySchedulerConfiguration newConf =
+        new CapacitySchedulerConfiguration();
+    newConf.set(getTemplateKey("root.a.*", "capacity"), "4w");
+    template =
+        new AutoCreatedQueueTemplate(newConf, TEST_QUEUE_AB);
+    template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC);
+
+    Assert.assertEquals("precedence is invalid", 4f,
+        newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+  }
+
+  private String getTemplateKey(String queuePath, String entryKey) {
+    return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+        + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey;
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 4dae4fd64eb3d..c514fc7af2a04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -647,6 +648,85 @@ public void testAutoCreateQueueMaxQueuesLimit() throws Exception {
     }
   }
 
+  @Test
+  public void testAutoCreatedQueueTemplateConfig() throws Exception {
+    startScheduler();
+    csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
+        "root.a.*") + "capacity", "6w");
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    LeafQueue a2 = createQueue("root.a.a-auto.a2");
+    Assert.assertEquals("weight is not set by template", 6f,
+        a2.getQueueCapacities().getWeight(), 1e-6);
+
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2");
+    Assert.assertEquals("weight is overridden", 6f,
+        a2.getQueueCapacities().getWeight(), 1e-6);
+
+    csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    Assert.assertEquals("weight is not explicitly set", 4f,
+        a2.getQueueCapacities().getWeight(), 1e-6);
+  }
+
+  @Test
+  public void testAutoCreatedQueueConfigChange() throws Exception {
+    startScheduler();
+    LeafQueue a2 = createQueue("root.a.a-auto.a2");
+    csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    Assert.assertEquals("weight is not explicitly set", 4f,
+        a2.getQueueCapacities().getWeight(), 1e-6);
+
+    a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2");
+    csConf.setState("root.a.a-auto.a2", QueueState.STOPPED);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    Assert.assertEquals("root.a.a-auto.a2 has not been stopped",
+        QueueState.STOPPED, a2.getState());
+
+    csConf.setState("root.a.a-auto.a2", QueueState.RUNNING);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    Assert.assertEquals("root.a.a-auto.a2 is not running",
+        QueueState.RUNNING, a2.getState());
+  }
+
+  @Test
+  public void testAutoCreateQueueState() throws Exception {
+    startScheduler();
+
+    createQueue("root.e.e1");
+    csConf.setState("root.e", QueueState.STOPPED);
+    csConf.setState("root.e.e1", QueueState.STOPPED);
+    csConf.setState("root.a", QueueState.STOPPED);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    // Make sure the static queue is stopped
+    Assert.assertEquals(cs.getQueue("root.a").getState(),
+        QueueState.STOPPED);
+    // If not set, default is the queue state of parent
+    Assert.assertEquals(cs.getQueue("root.a.a1").getState(),
+        QueueState.STOPPED);
+
+    Assert.assertEquals(cs.getQueue("root.e").getState(),
+        QueueState.STOPPED);
+    Assert.assertEquals(cs.getQueue("root.e.e1").getState(),
+        QueueState.STOPPED);
+
+    // Make root.e state to RUNNING
+    csConf.setState("root.e", QueueState.RUNNING);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    Assert.assertEquals(cs.getQueue("root.e.e1").getState(),
+        QueueState.STOPPED);
+
+    // Make root.e.e1 state to RUNNING
+    csConf.setState("root.e.e1", QueueState.RUNNING);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    Assert.assertEquals(cs.getQueue("root.e.e1").getState(),
+        QueueState.RUNNING);
+  }
+
   @Test
   public void testAutoQueueCreationDepthLimitFromStaticParent()
       throws Exception {

From 4c567fcff7af45c75117ee4a75c087aa454a89e5 Mon Sep 17 00:00:00 2001
From: Stephen O'Donnell <stephen.odonnell@gmail.com>
Date: Thu, 8 Apr 2021 11:59:02 +0100
Subject: [PATCH 10/43] HDFS-15937. Reduce memory used during datanode layout
 upgrade. Contributed by Stephen O'Donnell (#2838)

---
 .../hdfs/server/datanode/DataStorage.java     | 98 ++++++++++++-------
 1 file changed, 62 insertions(+), 36 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 4e216db892f67..03e99864a07f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1071,12 +1071,26 @@ private static void linkAllBlocks(File fromDir, File fromBbwDir, File toDir,
   }
 
   private static class LinkArgs {
-    File src;
-    File dst;
+    private File srcDir;
+    private File dstDir;
+    private String blockFile;
+
+    LinkArgs(File srcDir, File dstDir, String blockFile) {
+      this.srcDir = srcDir;
+      this.dstDir = dstDir;
+      this.blockFile = blockFile;
+    }
+
+    public File src() {
+      return new File(srcDir, blockFile);
+    }
 
-    LinkArgs(File src, File dst) {
-      this.src = src;
-      this.dst = dst;
+    public File dst() {
+      return new File(dstDir, blockFile);
+    }
+
+    public String blockFile() {
+      return blockFile;
     }
   }
 
@@ -1102,8 +1116,9 @@ private static void linkBlocks(File from, File to, int oldLV,
     }
 
     final ArrayList<LinkArgs> idBasedLayoutSingleLinks = Lists.newArrayList();
-    linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to,
-        idBasedLayoutSingleLinks);
+    final Map<File, File> pathCache = new HashMap<>();
+    linkBlocksHelper(from, to, hl, upgradeToIdBasedLayout, to,
+        idBasedLayoutSingleLinks, pathCache);
 
     // Detect and remove duplicate entries.
     final ArrayList<LinkArgs> duplicates =
@@ -1129,7 +1144,7 @@ public Void call() throws IOException {
               idBasedLayoutSingleLinks.size());
           for (int j = iCopy; j < upperBound; j++) {
             LinkArgs cur = idBasedLayoutSingleLinks.get(j);
-            HardLink.createHardLink(cur.src, cur.dst);
+            HardLink.createHardLink(cur.src(), cur.dst());
           }
           return null;
         }
@@ -1162,9 +1177,9 @@ static ArrayList<LinkArgs> findDuplicateEntries(ArrayList<LinkArgs> all) {
       @Override
       public int compare(LinkArgs a, LinkArgs b) {
         return ComparisonChain.start().
-            compare(a.src.getName(), b.src.getName()).
-            compare(a.src, b.src).
-            compare(a.dst, b.dst).
+            compare(a.blockFile(), b.blockFile()).
+            compare(a.src(), b.src()).
+            compare(a.dst(), b.dst()).
             result();
       }
     });
@@ -1174,8 +1189,8 @@ public int compare(LinkArgs a, LinkArgs b) {
     boolean addedPrev = false;
     for (int i = 0; i < all.size(); i++) {
       LinkArgs args = all.get(i);
-      long blockId = Block.getBlockId(args.src.getName());
-      boolean isMeta = Block.isMetaFilename(args.src.getName());
+      long blockId = Block.getBlockId(args.blockFile());
+      boolean isMeta = Block.isMetaFilename(args.blockFile());
       if ((prevBlockId == null) ||
           (prevBlockId.longValue() != blockId)) {
         prevBlockId = blockId;
@@ -1214,10 +1229,10 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
     TreeMap<Long, List<LinkArgs>> highestGenstamps =
         new TreeMap<Long, List<LinkArgs>>();
     for (LinkArgs duplicate : duplicates) {
-      if (!Block.isMetaFilename(duplicate.src.getName())) {
+      if (!Block.isMetaFilename(duplicate.blockFile())) {
         continue;
       }
-      long blockId = Block.getBlockId(duplicate.src.getName());
+      long blockId = Block.getBlockId(duplicate.blockFile());
       List<LinkArgs> prevHighest = highestGenstamps.get(blockId);
       if (prevHighest == null) {
         List<LinkArgs> highest = new LinkedList<LinkArgs>();
@@ -1226,8 +1241,8 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
         continue;
       }
       long prevGenstamp =
-          Block.getGenerationStamp(prevHighest.get(0).src.getName());
-      long genstamp = Block.getGenerationStamp(duplicate.src.getName());
+          Block.getGenerationStamp(prevHighest.get(0).blockFile());
+      long genstamp = Block.getGenerationStamp(duplicate.blockFile());
       if (genstamp < prevGenstamp) {
         continue;
       }
@@ -1241,19 +1256,19 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
     // from the duplicates list.
     for (Iterator<LinkArgs> iter = duplicates.iterator(); iter.hasNext(); ) {
       LinkArgs duplicate = iter.next();
-      long blockId = Block.getBlockId(duplicate.src.getName());
+      long blockId = Block.getBlockId(duplicate.blockFile());
       List<LinkArgs> highest = highestGenstamps.get(blockId);
       if (highest != null) {
         boolean found = false;
         for (LinkArgs high : highest) {
-          if (high.src.getParent().equals(duplicate.src.getParent())) {
+          if (high.src().getParent().equals(duplicate.src().getParent())) {
             found = true;
             break;
           }
         }
         if (!found) {
           LOG.warn("Unexpectedly low genstamp on {}.",
-              duplicate.src.getAbsolutePath());
+              duplicate.src().getAbsolutePath());
           iter.remove();
         }
       }
@@ -1264,25 +1279,25 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
     // preserving one block file / metadata file pair.
     TreeMap<Long, LinkArgs> longestBlockFiles = new TreeMap<Long, LinkArgs>();
     for (LinkArgs duplicate : duplicates) {
-      if (Block.isMetaFilename(duplicate.src.getName())) {
+      if (Block.isMetaFilename(duplicate.blockFile())) {
         continue;
       }
-      long blockId = Block.getBlockId(duplicate.src.getName());
+      long blockId = Block.getBlockId(duplicate.blockFile());
       LinkArgs prevLongest = longestBlockFiles.get(blockId);
       if (prevLongest == null) {
         longestBlockFiles.put(blockId, duplicate);
         continue;
       }
-      long blockLength = duplicate.src.length();
-      long prevBlockLength = prevLongest.src.length();
+      long blockLength = duplicate.src().length();
+      long prevBlockLength = prevLongest.src().length();
       if (blockLength < prevBlockLength) {
         LOG.warn("Unexpectedly short length on {}.",
-            duplicate.src.getAbsolutePath());
+            duplicate.src().getAbsolutePath());
         continue;
       }
       if (blockLength > prevBlockLength) {
         LOG.warn("Unexpectedly short length on {}.",
-            prevLongest.src.getAbsolutePath());
+            prevLongest.src().getAbsolutePath());
       }
       longestBlockFiles.put(blockId, duplicate);
     }
@@ -1291,21 +1306,22 @@ private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
     // arbitrarily selected by us.
     for (Iterator<LinkArgs> iter = all.iterator(); iter.hasNext(); ) {
       LinkArgs args = iter.next();
-      long blockId = Block.getBlockId(args.src.getName());
+      long blockId = Block.getBlockId(args.blockFile());
       LinkArgs bestDuplicate = longestBlockFiles.get(blockId);
       if (bestDuplicate == null) {
         continue; // file has no duplicates
       }
-      if (!bestDuplicate.src.getParent().equals(args.src.getParent())) {
-        LOG.warn("Discarding {}.", args.src.getAbsolutePath());
+      if (!bestDuplicate.src().getParent().equals(args.src().getParent())) {
+        LOG.warn("Discarding {}.", args.src().getAbsolutePath());
         iter.remove();
       }
     }
   }
 
-  static void linkBlocksHelper(File from, File to, int oldLV, HardLink hl,
-  boolean upgradeToIdBasedLayout, File blockRoot,
-      List<LinkArgs> idBasedLayoutSingleLinks) throws IOException {
+  static void linkBlocksHelper(File from, File to, HardLink hl,
+      boolean upgradeToIdBasedLayout, File blockRoot,
+      List<LinkArgs> idBasedLayoutSingleLinks, Map<File, File> pathCache)
+      throws IOException {
     if (!from.exists()) {
       return;
     }
@@ -1345,8 +1361,18 @@ public boolean accept(File dir, String name) {
               throw new IOException("Failed to mkdirs " + blockLocation);
             }
           }
-          idBasedLayoutSingleLinks.add(new LinkArgs(new File(from, blockName),
-              new File(blockLocation, blockName)));
+          /**
+           * The destination path is 32x32, so 1024 distinct paths. Therefore
+           * we cache the destination path and reuse the same File object on
+           * potentially thousands of blocks located on this volume.
+           * This method is called recursively so the cache is passed through
+           * each recursive call. There is one cache per volume, and it is only
+           * accessed by a single thread so no locking is needed.
+           */
+          File cachedDest = pathCache
+              .computeIfAbsent(blockLocation, k -> blockLocation);
+          idBasedLayoutSingleLinks.add(new LinkArgs(from,
+              cachedDest, blockName));
           hl.linkStats.countSingleLinks++;
         }
       } else {
@@ -1369,8 +1395,8 @@ public boolean accept(File dir, String name) {
     if (otherNames != null) {
       for (int i = 0; i < otherNames.length; i++) {
         linkBlocksHelper(new File(from, otherNames[i]),
-            new File(to, otherNames[i]), oldLV, hl, upgradeToIdBasedLayout,
-            blockRoot, idBasedLayoutSingleLinks);
+            new File(to, otherNames[i]), hl, upgradeToIdBasedLayout,
+            blockRoot, idBasedLayoutSingleLinks, pathCache);
       }
     }
   }

From c6539e3289711d29f508930bbda40302f48ddf4c Mon Sep 17 00:00:00 2001
From: Ayush Saxena <ayushsaxena@apache.org>
Date: Thu, 8 Apr 2021 20:49:08 +0530
Subject: [PATCH 11/43] HDFS-15916. DistCp: Backward compatibility: Distcp
 fails from Hadoop 3 to Hadoop 2 for snapshotdiff. (#2863). Contributed by
 Ayush Saxena.

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
---
 .../apache/hadoop/hdfs/DistributedFileSystem.java    | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index fe2d077977ca2..eda92be12c627 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdfs;
 
 
+import org.apache.hadoop.ipc.RpcNoSuchMethodException;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
@@ -2388,8 +2389,15 @@ private SnapshotDiffReport getSnapshotDiffReportInternal(
     List<DiffReportListingEntry> deletedList = new ChunkedArrayList<>();
     SnapshotDiffReportListing report;
     do {
-      report = dfs.getSnapshotDiffReportListing(snapshotDir, fromSnapshot,
-          toSnapshot, startPath, index);
+      try {
+        report = dfs.getSnapshotDiffReportListing(snapshotDir, fromSnapshot,
+            toSnapshot, startPath, index);
+      } catch (RpcNoSuchMethodException e) {
+        // In case the server doesn't support getSnapshotDiffReportListing,
+        // fallback to getSnapshotDiffReport.
+        LOG.warn("Falling back to getSnapshotDiffReport {}", e.getMessage());
+        return dfs.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot);
+      }
       startPath = report.getLastPath();
       index = report.getLastIndex();
       modifiedList.addAll(report.getModifyList());

From bf661164077c002d9ddc9fc26ba2d89f1623ecb3 Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Thu, 8 Apr 2021 22:14:47 +0530
Subject: [PATCH 12/43] HDFS-15955. Make explicit_bzero cross platform (#2875)

---
 .../native/libhdfspp/lib/bindings/c/hdfs.cc   |  3 ++-
 .../native/libhdfspp/lib/x-platform/syscall.h | 12 ++++++++++
 .../libhdfspp/lib/x-platform/syscall_linux.cc |  7 ++++++
 .../lib/x-platform/syscall_windows.cc         |  8 +++++++
 .../native/libhdfspp/tests/hdfs_ext_test.cc   |  3 ++-
 .../native/libhdfspp/tests/hdfspp_mini_dfs.h  |  3 ++-
 .../tests/x-platform/syscall_common_test.cc   | 23 +++++++++++++++++++
 7 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index efa4c750108b7..80f9316160216 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -25,6 +25,7 @@
 #include "fs/filesystem.h"
 #include "fs/filehandle.h"
 #include "x-platform/utils.h"
+#include "x-platform/syscall.h"
 
 #include <limits.h>
 #include <string>
@@ -1395,7 +1396,7 @@ int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations
     hdfsBlockLocations *locations = new struct hdfsBlockLocations();
     (*locations_out) = locations;
 
-    explicit_bzero(locations, sizeof(*locations));
+    XPlatform::Syscall::ClearBufferSafely(locations, sizeof(*locations));
     locations->fileLength = ppLocations->getFileLength();
     locations->isLastBlockComplete = ppLocations->isLastBlockComplete();
     locations->isUnderConstruction = ppLocations->isUnderConstruction();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
index 297acebfc5c8d..4f94ecbe31dfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
@@ -59,6 +59,18 @@ class Syscall {
    */
   static bool FnMatch(const std::string& pattern, const std::string& str);
 
+  /**
+   * Clears the given {@link buffer} upto {@link sz_bytes} by
+   * filling them with zeros. This method is immune to compiler
+   * optimizations and guarantees that the first {@link sz_bytes} of
+   * {@link buffer} is cleared. The {@link buffer} must be at least
+   * as big as {@link sz_bytes}, the behaviour is undefined otherwise.
+   *
+   * @param buffer the pointer to the buffer to clear.
+   * @param sz_bytes the count of the bytes to clear.
+   */
+  static void ClearBufferSafely(void* buffer, size_t sz_bytes);
+
  private:
   static bool WriteToStdoutImpl(const char* message);
 };
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index 2c51dbfddfca6..9821cc7110b2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -41,3 +41,10 @@ bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
   const auto result = write(1, message, message_len);
   return result == static_cast<ssize_t>(message_len);
 }
+
+void XPlatform::Syscall::ClearBufferSafely(void* buffer,
+                                           const size_t sz_bytes) {
+  if (buffer != nullptr) {
+    explicit_bzero(buffer, sz_bytes);
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
index dc9ba63634f1b..5a3423a99f196 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
@@ -17,6 +17,7 @@
  */
 
 #include <Shlwapi.h>
+#include <WinBase.h>
 #include <Windows.h>
 
 #include "syscall.h"
@@ -49,3 +50,10 @@ bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
       WriteFile(stdout_handle, message, message_len, &bytes_written, nullptr);
   return result && static_cast<unsigned long>(message_len) == bytes_written;
 }
+
+void XPlatform::Syscall::ClearBufferSafely(void* buffer,
+                                           const size_t sz_bytes) {
+  if (buffer != nullptr) {
+    SecureZeroMemory(buffer, sz_bytes);
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
index fb55172633af0..34e53842b1605 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc
@@ -18,6 +18,7 @@
 
 #include "hdfspp_mini_dfs.h"
 #include "hdfspp/hdfs_ext.h"
+#include "x-platform/syscall.h"
 
 #include <cstring>
 #include <chrono>
@@ -475,7 +476,7 @@ TEST_F(HdfsExtTest, TestReadStats) {
   hdfsFile file = hdfsOpenFile(fs, path.c_str(), O_WRONLY, 0, 0, 0);
   EXPECT_NE(nullptr, file);
   void * buf = malloc(size);
-  explicit_bzero(buf, size);
+  XPlatform::Syscall::ClearBufferSafely(buf, size);
   EXPECT_EQ(size, hdfsWrite(fs, file, buf, size));
   free(buf);
   EXPECT_EQ(0, hdfsCloseFile(fs, file));
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h
index 98edbdc1d6501..aae8d83563f1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h
@@ -19,6 +19,7 @@
 #include "hdfs/hdfs.h"
 #include "hdfspp/hdfspp.h"
 #include <native_mini_dfs.h>
+#include "x-platform/syscall.h"
 
 #include <google/protobuf/io/coded_stream.h>
 #include <gmock/gmock.h>
@@ -92,7 +93,7 @@ class HdfsHandle {
     hdfsFile file = hdfsOpenFile(*this, path.c_str(), O_WRONLY, 0, 0, 0);
     EXPECT_NE(nullptr, file);
     void * buf = malloc(size);
-    explicit_bzero(buf, size);
+    XPlatform::Syscall::ClearBufferSafely(buf, size);
     EXPECT_EQ(1024, hdfsWrite(*this, file, buf, size));
     EXPECT_EQ(0, hdfsCloseFile(*this, file));
     free(buf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
index 04da29a33fb5c..d68b2afef9eea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
@@ -18,7 +18,9 @@
 
 #include <gtest/gtest.h>
 
+#include <numeric>
 #include <string>
+#include <vector>
 
 #include "x-platform/syscall.h"
 
@@ -45,3 +47,24 @@ TEST(XPlatformSyscall, FnMatchNegativeQuestionMark) {
   const std::string str("abc.doc");
   EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
 }
+
+TEST(XPlatformSyscall, ClearBufferSafelyChars) {
+  std::vector<char> alphabets(26);
+  std::iota(alphabets.begin(), alphabets.end(), 'a');
+
+  XPlatform::Syscall::ClearBufferSafely(alphabets.data(), alphabets.size());
+  for (const auto alphabet : alphabets) {
+    EXPECT_EQ(alphabet, '\0');
+  }
+}
+
+TEST(XPlatformSyscall, ClearBufferSafelyNumbers) {
+  std::vector<int> numbers(200);
+  std::iota(numbers.begin(), numbers.end(), 0);
+
+  XPlatform::Syscall::ClearBufferSafely(numbers.data(),
+                                        numbers.size() * sizeof(int));
+  for (const auto number : numbers) {
+    EXPECT_EQ(number, 0);
+  }
+}
\ No newline at end of file

From 213d3deb2621d3c22ab8bc507fffb38fe8bb42a1 Mon Sep 17 00:00:00 2001
From: Eric Badger <ebadger@verizonmedia.com>
Date: Fri, 9 Apr 2021 00:34:15 +0000
Subject: [PATCH 13/43] YARN-10503. Support queue capacity in terms of absolute
 resources with custom resourceType. Contributed by Qi Zhu.

---
 .../yarn/util/resource/ResourceUtils.java     | 15 ++++
 .../CapacitySchedulerConfiguration.java       | 35 ++++++--
 .../TestCSAllocateCustomResource.java         | 82 +++++++++++++++++++
 .../test/resources/resource-types-test.xml    |  2 +-
 4 files changed, 126 insertions(+), 8 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index f7c75a6079ffb..3654965c05df1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -902,4 +902,19 @@ private static void validateResourceTypes(
       }
     }
   }
+
+  public static StringBuilder
+      getCustomResourcesStrings(Resource resource) {
+    StringBuilder res = new StringBuilder();
+    if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+      ResourceInformation[] resources =
+          resource.getResources();
+      for (int i = 2; i < resources.length; i++) {
+        ResourceInformation resInfo = resources[i];
+        res.append(","
+            + resInfo.getName() + "=" + resInfo.getValue());
+      }
+    }
+    return  res;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 266cbb4d34601..074e3711ba757 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -2350,11 +2350,14 @@ public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath,
         getAutoCreatedQueueTemplateConfPrefix(queuePath);
 
     StringBuilder resourceString = new StringBuilder();
+
     resourceString
         .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
             + resource.getMemorySize() + ","
             + AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
-            + resource.getVirtualCores() + "]");
+            + resource.getVirtualCores()
+            + ResourceUtils.
+            getCustomResourcesStrings(resource) + "]");
 
     setCapacityByLabel(leafQueueConfPrefix, label, resourceString.toString());
   }
@@ -2385,11 +2388,14 @@ public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath,
         queuePath);
 
     StringBuilder resourceString = new StringBuilder();
+
     resourceString
         .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
             + resource.getMemorySize() + ","
             + AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
-            + resource.getVirtualCores() + "]");
+            + resource.getVirtualCores()
+            + ResourceUtils.
+            getCustomResourcesStrings(resource) + "]");
 
     setMaximumCapacityByLabel(leafQueueConfPrefix, label, resourceString.toString());
   }
@@ -2489,11 +2495,14 @@ private void updateMinMaxResourceToConf(String label, String queue,
     }
 
     StringBuilder resourceString = new StringBuilder();
+
     resourceString
         .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
             + resource.getMemorySize() + ","
             + AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
-            + resource.getVirtualCores() + "]");
+            + resource.getVirtualCores()
+            + ResourceUtils.
+            getCustomResourcesStrings(resource) + "]");
 
     String prefix = getQueuePrefix(queue) + type;
     if (!label.isEmpty()) {
@@ -2567,8 +2576,12 @@ private Resource internalGetLabeledResourceRequirementForQueue(String queue,
   private void updateResourceValuesFromConfig(Set<String> resourceTypes,
       Resource resource, String[] splits) {
 
+    String resourceName = splits[0].trim();
+
     // If key is not a valid type, skip it.
-    if (!resourceTypes.contains(splits[0])) {
+    if (!resourceTypes.contains(resourceName)
+        && !ResourceUtils.getResourceTypes().containsKey(resourceName)) {
+      LOG.error(resourceName + " not supported.");
       return;
     }
 
@@ -2581,9 +2594,17 @@ private void updateResourceValuesFromConfig(Set<String> resourceTypes,
       resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
     }
 
+    // Custom resource type defined by user.
+    // Such as GPU FPGA etc.
+    if (!resourceTypes.contains(resourceName)) {
+      resource.setResourceInformation(resourceName, ResourceInformation
+          .newInstance(resourceName, units, resourceValue));
+      return;
+    }
+
     // map it based on key.
     AbsoluteResourceType resType = AbsoluteResourceType
-        .valueOf(StringUtils.toUpperCase(splits[0].trim()));
+        .valueOf(StringUtils.toUpperCase(resourceName));
     switch (resType) {
     case MEMORY :
       resource.setMemorySize(resourceValue);
@@ -2592,8 +2613,8 @@ private void updateResourceValuesFromConfig(Set<String> resourceTypes,
       resource.setVirtualCores(resourceValue.intValue());
       break;
     default :
-      resource.setResourceInformation(splits[0].trim(), ResourceInformation
-          .newInstance(splits[0].trim(), units, resourceValue));
+      resource.setResourceInformation(resourceName, ResourceInformation
+          .newInstance(resourceName, units, resourceValue));
       break;
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
index 7b0254cdcc5da..36b3c9b4d63ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -50,10 +51,16 @@
 
 import java.io.File;
 import java.io.IOException;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
+import java.util.Map;
+import java.util.Arrays;
+import java.util.stream.Collectors;
 
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.FPGA_URI;
 import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
 import static org.junit.Assert.assertEquals;
@@ -248,4 +255,79 @@ public void testClusterMetricsWithGPU()
             .get(GPU_URI)).longValue(), 0);
     ClusterMetrics.destroy();
   }
+
+  /**
+   * Test CS absolute conf with Custom resource type.
+   * */
+  @Test
+  public void testCapacitySchedulerAbsoluteConfWithCustomResourceType()
+      throws IOException {
+    // reset resource types
+    ResourceUtils.resetResourceTypes();
+    String resourceTypesFileName = "resource-types-test.xml";
+    File source = new File(
+        conf.getClassLoader().getResource(resourceTypesFileName).getFile());
+    resourceTypesFile = new File(source.getParent(), "resource-types.xml");
+    FileUtils.copyFile(source, resourceTypesFile);
+
+    CapacitySchedulerConfiguration newConf =
+        new CapacitySchedulerConfiguration(conf);
+
+    // Only memory vcores for first class.
+    Set<String> resourceTypes = Arrays.
+        stream(CapacitySchedulerConfiguration.
+            AbsoluteResourceType.values()).
+        map(value -> value.toString().toLowerCase()).
+        collect(Collectors.toSet());
+
+    Map<String, Long> valuesMin = Maps.newHashMap();
+    valuesMin.put(GPU_URI, 10L);
+    valuesMin.put(FPGA_URI, 10L);
+    valuesMin.put("testType", 10L);
+
+    Map<String, Long> valuesMax = Maps.newHashMap();
+    valuesMax.put(GPU_URI, 100L);
+    valuesMax.put(FPGA_URI, 100L);
+    valuesMax.put("testType", 100L);
+
+    Resource aMINRES =
+        Resource.newInstance(1000, 10, valuesMin);
+
+    Resource aMAXRES =
+        Resource.newInstance(1000, 10, valuesMax);
+
+    // Define top-level queues
+    newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[] {"a", "b", "c"});
+    newConf.setMinimumResourceRequirement("", "root.a",
+        aMINRES);
+    newConf.setMaximumResourceRequirement("", "root.a",
+        aMAXRES);
+
+    newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+        DominantResourceCalculator.class, ResourceCalculator.class);
+
+    //start RM
+    MockRM rm = new MockRM(newConf);
+    rm.start();
+
+    // Check the gpu resource conf is right.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    Assert.assertEquals(aMINRES,
+        cs.getConfiguration().
+            getMinimumResourceRequirement("", "root.a", resourceTypes));
+    Assert.assertEquals(aMAXRES,
+        cs.getConfiguration().
+            getMaximumResourceRequirement("", "root.a", resourceTypes));
+
+    // Check the gpu resource of queue is right.
+    Assert.assertEquals(aMINRES, cs.getQueue("root.a").
+        getQueueResourceQuotas().getConfiguredMinResource());
+    Assert.assertEquals(aMAXRES, cs.getQueue("root.a").
+        getQueueResourceQuotas().getConfiguredMaxResource());
+
+    rm.close();
+
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml
index be4f07428c521..2ae7531172b4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/resource-types-test.xml
@@ -17,6 +17,6 @@
 <configuration>
     <property>
         <name>yarn.resource-types</name>
-        <value>yarn.io/gpu, yarn.io/fpga</value>
+        <value>yarn.io/gpu, yarn.io/fpga, testType</value>
     </property>
 </configuration>
\ No newline at end of file

From 56bd968fb434c7d2ae7ac9e7659d0ca682340b2a Mon Sep 17 00:00:00 2001
From: Viraj Jasani <vjasani@apache.org>
Date: Fri, 9 Apr 2021 07:40:18 +0530
Subject: [PATCH 14/43] HDFS-15940. Fix
 TestBlockRecovery2#testRaceBetweenReplicaRecoveryAndFinalizeBlock (ADDENDUM)
 (#2874)

---
 .../hadoop/hdfs/server/datanode/TestBlockRecovery2.java      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java
index 03d5851f23257..8d2df18711256 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java
@@ -229,8 +229,8 @@ public void testRaceBetweenReplicaRecoveryAndFinalizeBlock()
     tearDown();
 
     Configuration configuration = new HdfsConfiguration();
-    configuration.set(
-        DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, "1000");
+    configuration.setLong(
+        DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, 5000L);
     MiniDFSCluster cluster = new MiniDFSCluster.Builder(configuration)
         .numDataNodes(1).build();
     try {
@@ -257,6 +257,7 @@ public void testRaceBetweenReplicaRecoveryAndFinalizeBlock()
             dataNode.initReplicaRecovery(recoveringBlock);
           }
         } catch (Exception e) {
+          LOG.error("Something went wrong.", e);
           recoveryInitResult.set(false);
         }
       });

From 663ca14a769bd8fa124c1aff4ac6630491dbb425 Mon Sep 17 00:00:00 2001
From: lichaojacobs <lichaojacobs@tju.edu.cn>
Date: Fri, 9 Apr 2021 10:58:53 +0800
Subject: [PATCH 15/43] MAPREDUCE-7329: HadoopPipes task may fail when linux
 kernel version change from 3.x to 4.x (#2775)

---
 .../hadoop/mapred/pipes/Application.java      | 52 ++++++++++++
 .../mapred/pipes/TestPipeApplication.java     | 83 +++++++++++++++++++
 2 files changed, 135 insertions(+)

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
index 83d250981f71e..5416d2693682d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
@@ -30,12 +30,14 @@
 
 import javax.crypto.SecretKey;
 
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
@@ -52,6 +54,7 @@
 import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
@@ -66,6 +69,7 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
   private static final Logger LOG =
       LoggerFactory.getLogger(Application.class.getName());
   private ServerSocket serverSocket;
+  private PingSocketCleaner socketCleaner;
   private Process process;
   private Socket clientSocket;
   private OutputHandler<K2, V2> handler;
@@ -133,6 +137,13 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
     
     process = runClient(cmd, env);
     clientSocket = serverSocket.accept();
+    // start ping socket cleaner
+    int soTimeout = conf.getInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,
+        CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT);
+    socketCleaner = new PingSocketCleaner("ping-socket-cleaner", serverSocket,
+                                          soTimeout);
+    socketCleaner.setDaemon(true);
+    socketCleaner.start();
     
     String challenge = getSecurityChallenge();
     String digestToSend = createDigest(password, challenge);
@@ -237,6 +248,7 @@ void cleanup() throws IOException {
     serverSocket.close();
     try {
       downlink.close();
+      socketCleaner.interrupt();
     } catch (InterruptedException ie) {
       Thread.currentThread().interrupt();
     }      
@@ -266,4 +278,44 @@ public static String createDigest(byte[] password, String data)
     return SecureShuffleUtils.hashFromString(data, key);
   }
 
+  @VisibleForTesting
+  public static class PingSocketCleaner extends Thread {
+    private final ServerSocket serverSocket;
+    private final int soTimeout;
+
+    PingSocketCleaner(String name, ServerSocket serverSocket, int soTimeout) {
+      super(name);
+      this.serverSocket = serverSocket;
+      this.soTimeout = soTimeout;
+    }
+
+    @Override
+    public void run() {
+      LOG.info("PingSocketCleaner started...");
+      while (!Thread.currentThread().isInterrupted()) {
+        Socket clientSocket = null;
+        try {
+          clientSocket = serverSocket.accept();
+          clientSocket.setSoTimeout(soTimeout);
+          LOG.debug("Connection received from {}",
+                    clientSocket.getInetAddress());
+          int readData = 0;
+          while (readData != -1) {
+            readData = clientSocket.getInputStream().read();
+          }
+          LOG.debug("close socket cause client has closed.");
+          closeSocketInternal(clientSocket);
+        } catch (IOException exception) {
+          LOG.error("PingSocketCleaner exception", exception);
+        } finally {
+          closeSocketInternal(clientSocket);
+        }
+      }
+    }
+
+    @VisibleForTesting
+    protected void closeSocketInternal(Socket clientSocket) {
+      IOUtils.closeSocket(clientSocket);
+    }
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
index f0b383af5f830..79c0bc12222c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
@@ -28,12 +28,15 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintStream;
+import java.net.ServerSocket;
+import java.net.Socket;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.FsConstants;
@@ -59,7 +62,9 @@
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.TaskAttemptID;
 import org.apache.hadoop.mapred.TaskLog;
+import org.apache.hadoop.mapred.pipes.Application.PingSocketCleaner;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -455,6 +460,84 @@ public void testPipesPartitioner() {
     assertEquals(3, partitioner.getPartition(iw, new Text("test"), 2));
   }
 
+  @Test
+  public void testSocketCleaner() throws Exception {
+    ServerSocket serverSocket = setupServerSocket();
+    SocketCleaner cleaner = setupCleaner(serverSocket);
+    // mock ping thread, connect to server socket per second.
+    int expectedClosedCount = 5;
+    for (int i = 0; i < expectedClosedCount; i++) {
+      try {
+        Thread.sleep(1000);
+        Socket clientSocket = new Socket(serverSocket.getInetAddress(),
+                                         serverSocket.getLocalPort());
+        clientSocket.close();
+      } catch (Exception exception) {
+        // ignored...
+        exception.printStackTrace();
+      }
+    }
+    GenericTestUtils.waitFor(
+        () -> expectedClosedCount == cleaner.getCloseSocketCount(), 100, 5000);
+  }
+
+  @Test
+  public void testSocketTimeout() throws Exception {
+    ServerSocket serverSocket = setupServerSocket();
+    SocketCleaner cleaner = setupCleaner(serverSocket, 100);
+    try {
+      new Socket(serverSocket.getInetAddress(), serverSocket.getLocalPort());
+      Thread.sleep(1000);
+    } catch (Exception exception) {
+      // ignored...
+    }
+    GenericTestUtils.waitFor(() -> 1 == cleaner.getCloseSocketCount(), 100,
+        5000);
+  }
+
+  private SocketCleaner setupCleaner(ServerSocket serverSocket) {
+    return setupCleaner(serverSocket,
+                        CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT);
+  }
+
+  private SocketCleaner setupCleaner(ServerSocket serverSocket, int soTimeout) {
+    // start socket cleaner.
+    SocketCleaner cleaner = new SocketCleaner("test-ping-socket-cleaner",
+                                              serverSocket, soTimeout);
+    cleaner.setDaemon(true);
+    cleaner.start();
+
+    return cleaner;
+  }
+
+  private static class SocketCleaner extends PingSocketCleaner {
+    private int closeSocketCount = 0;
+
+    SocketCleaner(String name, ServerSocket serverSocket, int soTimeout) {
+      super(name, serverSocket, soTimeout);
+    }
+
+    @Override
+    public void run() {
+      super.run();
+    }
+
+    protected void closeSocketInternal(Socket clientSocket) {
+      if (!clientSocket.isClosed()) {
+        closeSocketCount++;
+      }
+      super.closeSocketInternal(clientSocket);
+    }
+
+    public int getCloseSocketCount() {
+      return closeSocketCount;
+    }
+  }
+
+  private ServerSocket setupServerSocket() throws Exception {
+    return new ServerSocket(0, 1);
+  }
+
   /**
    * clean previous std error and outs
    */

From 6f640abbaf14efa98d6c599e5fff95647730ad42 Mon Sep 17 00:00:00 2001
From: sumangala-patki <70206833+sumangala-patki@users.noreply.github.com>
Date: Fri, 9 Apr 2021 09:31:23 +0530
Subject: [PATCH 16/43] HADOOP-17576. ABFS: Disable throttling update for auth
 failures (#2761)

Contributed by Sumangala Patki
---
 .../azurebfs/services/AbfsRestOperation.java  | 24 ++++------
 .../ITestAbfsRestOperationException.java      | 47 +++++++++++++++++--
 2 files changed, 52 insertions(+), 19 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index 24ec2926647e6..584b71f1ee5bf 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -34,7 +34,6 @@
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidAbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
-import org.apache.hadoop.fs.azurebfs.oauth2.AzureADAuthenticator.HttpException;
 
 /**
  * The AbfsRestOperation for Rest AbfsClient.
@@ -233,7 +232,13 @@ private boolean executeHttpOperation(final int retryCount) throws AzureBlobFileS
               hasRequestBody ? bufferLength : 0);
           break;
       }
+    } catch (IOException e) {
+      LOG.debug("Auth failure: {}, {}", method, url);
+      throw new AbfsRestOperationException(-1, null,
+          "Auth failure: " + e.getMessage(), e);
+    }
 
+    try {
       // dump the headers
       AbfsIoUtils.dumpHeadersToDebugLog("Request Headers",
           httpOperation.getConnection().getRequestProperties());
@@ -256,9 +261,7 @@ private boolean executeHttpOperation(final int retryCount) throws AzureBlobFileS
       }
     } catch (UnknownHostException ex) {
       String hostname = null;
-      if (httpOperation != null) {
-        hostname = httpOperation.getHost();
-      }
+      hostname = httpOperation.getHost();
       LOG.warn("Unknown host name: %s. Retrying to resolve the host name...",
           hostname);
       if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) {
@@ -267,24 +270,13 @@ private boolean executeHttpOperation(final int retryCount) throws AzureBlobFileS
       return false;
     } catch (IOException ex) {
       if (LOG.isDebugEnabled()) {
-        if (httpOperation != null) {
-          LOG.debug("HttpRequestFailure: " + httpOperation.toString(), ex);
-        } else {
-          LOG.debug("HttpRequestFailure: " + method + "," + url, ex);
-        }
+        LOG.debug("HttpRequestFailure: {}, {}", httpOperation.toString(), ex);
       }
 
       if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) {
         throw new InvalidAbfsRestOperationException(ex);
       }
 
-      // once HttpException is thrown by AzureADAuthenticator,
-      // it indicates the policy in AzureADAuthenticator determined
-      // retry is not needed
-      if (ex instanceof HttpException) {
-        throw new AbfsRestOperationException((HttpException) ex);
-      }
-
       return false;
     } finally {
       AbfsClientThrottlingIntercept.updateMetrics(operationType, httpOperation);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index 1d86de7ebeb5d..a71e7bc815f75 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -20,21 +20,31 @@
 
 import java.io.IOException;
 
+import org.assertj.core.api.Assertions;
+import org.junit.Assert;
+import org.junit.Test;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
 import org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import org.junit.Assert;
-import org.junit.Test;
-
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Verify the AbfsRestOperationException error message format.
  * */
 public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest{
+  private static final String RETRY_TEST_TOKEN_PROVIDER = "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider";
+
   public ITestAbfsRestOperationException() throws Exception {
     super();
   }
@@ -114,4 +124,35 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
             + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries
             + ")", RetryTestTokenProvider.reTryCount == numOfRetries);
   }
+
+  @Test
+  public void testAuthFailException() throws Exception {
+    Configuration config = new Configuration(getRawConfiguration());
+    String accountName = config
+        .get(FS_AZURE_ABFS_ACCOUNT_NAME);
+    // Setup to configure custom token provider
+    config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT
+        + accountName, "Custom");
+    config.set(
+        FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName,
+        RETRY_TEST_TOKEN_PROVIDER);
+    // Stop filesystem creation as it will lead to calls to store.
+    config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
+
+    final AzureBlobFileSystem fs = getFileSystem(config);
+    try {
+      fs.getFileStatus(new Path("/"));
+      fail("Should fail at auth token fetch call");
+    } catch (AbfsRestOperationException e) {
+      String errorDesc = "Should throw RestOp exception on AAD failure";
+      Assertions.assertThat(e.getStatusCode())
+          .describedAs("Incorrect status code. " + errorDesc).isEqualTo(-1);
+      Assertions.assertThat(e.getErrorCode())
+          .describedAs("Incorrect error code. " + errorDesc)
+          .isEqualTo(AzureServiceErrorCode.UNKNOWN);
+      Assertions.assertThat(e.getErrorMessage())
+          .describedAs("Incorrect error message. " + errorDesc)
+          .contains("Auth failure: ");
+    }
+  }
 }
\ No newline at end of file

From 1448756505f95624317fb746bf4b8406b7cb3419 Mon Sep 17 00:00:00 2001
From: Akira Ajisaka <aajisaka@apache.org>
Date: Fri, 9 Apr 2021 15:53:51 +0900
Subject: [PATCH 17/43] MAPREDUCE-7334. TestJobEndNotifier fails. (#2877)

---
 .../test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
index 75893f5660699..e3d1241cdc679 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
@@ -80,7 +80,7 @@ public void doGet(HttpServletRequest request,
       calledTimes++;
       try {
         // Sleep for a long time
-        Thread.sleep(1000000);
+        Thread.sleep(3000);
       } catch (InterruptedException e) {
         timedOut = true;
       }

From 3148791da42b48db0ecb611db85eda39a7f7d452 Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Fri, 9 Apr 2021 21:31:33 +0530
Subject: [PATCH 18/43] HDFS-15962. Make strcasecmp cross platform (#2883)

---
 .../libhdfspp/lib/common/CMakeLists.txt       |  6 +++--
 .../libhdfspp/lib/common/configuration.cc     |  5 ++--
 .../lib/common/configuration_loader.cc        |  7 ++---
 .../native/libhdfspp/lib/fs/filesystem.cc     |  2 +-
 .../native/libhdfspp/lib/rpc/CMakeLists.txt   |  5 ++--
 .../native/libhdfspp/lib/rpc/sasl_protocol.cc | 26 +++++++++----------
 .../native/libhdfspp/lib/x-platform/syscall.h | 13 ++++++++++
 .../libhdfspp/lib/x-platform/syscall_linux.cc |  6 +++++
 .../lib/x-platform/syscall_windows.cc         |  7 +++++
 .../tests/x-platform/syscall_common_test.cc   | 19 +++++++++++++-
 10 files changed, 71 insertions(+), 25 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
index 87779e7f8ae81..6bd7a266fbf51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt
@@ -20,6 +20,8 @@ if(NEED_LINK_DL)
 endif()
 
 include_directories(${Boost_INCLUDE_DIRS} ../../include)
-add_library(common_obj OBJECT status.cc sasl_digest_md5.cc ioservice_impl.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc config_parser.cc)
-add_library(common $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
+add_library(common_obj OBJECT $<TARGET_OBJECTS:x_platform_obj> status.cc sasl_digest_md5.cc ioservice_impl.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc uri.cc util.cc retry_policy.cc cancel_tracker.cc logging.cc libhdfs_events_impl.cc auth_info.cc namenode_info.cc statinfo.cc fsinfo.cc content_summary.cc locks.cc config_parser.cc)
+add_library(common $<TARGET_OBJECTS:x_platform_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
 target_link_libraries(common ${LIB_DL})
+target_include_directories(common_obj PRIVATE ../../lib)
+target_include_directories(common PRIVATE ../../lib)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
index 298de1e3aabfe..947214bdbd50a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
@@ -33,6 +33,7 @@
 
 #include "configuration.h"
 #include "hdfspp/uri.h"
+#include "x-platform/syscall.h"
 
 #include <strings.h>
 #include <sstream>
@@ -124,10 +125,10 @@ optional<bool> Configuration::GetBool(const std::string& key) const {
     return optional<bool>();
   }
 
-  if (!strcasecmp(raw->c_str(), "true")) {
+  if (XPlatform::Syscall::StringCompareIgnoreCase(*raw, "true")) {
     return std::experimental::make_optional(true);
   }
-  if (!strcasecmp(raw->c_str(), "false")) {
+  if (XPlatform::Syscall::StringCompareIgnoreCase(*raw, "false")) {
     return std::experimental::make_optional(false);
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
index 691d2ff719542..5301137505af1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
@@ -18,6 +18,7 @@
 
 #include "configuration_loader.h"
 #include "common/logging.h"
+#include "x-platform/syscall.h"
 
 #include <fstream>
 #include <strings.h>
@@ -46,17 +47,17 @@ bool is_valid_bool(const std::string& raw) {
     return false;
   }
 
-  if (!strcasecmp(raw.c_str(), "true")) {
+  if (XPlatform::Syscall::StringCompareIgnoreCase(raw, "true")) {
     return true;
   }
-  if (!strcasecmp(raw.c_str(), "false")) {
+  if (XPlatform::Syscall::StringCompareIgnoreCase(raw, "false")) {
     return true;
   }
   return false;
 }
 
 bool str_to_bool(const std::string& raw) {
-  if (!strcasecmp(raw.c_str(), "true")) {
+  if (XPlatform::Syscall::StringCompareIgnoreCase(raw, "true")) {
     return true;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index 741d6c783b62e..e92a9ee48d6b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -298,7 +298,7 @@ void FileSystemImpl::Connect(const std::string &server,
 
 void FileSystemImpl::ConnectToDefaultFs(const std::function<void(const Status &, FileSystem *)> &handler) {
   std::string scheme = options_.defaultFS.get_scheme();
-  if (strcasecmp(scheme.c_str(), "hdfs") != 0) {
+  if (!XPlatform::Syscall::StringCompareIgnoreCase(scheme, "hdfs")) {
     std::string error_message;
     error_message += "defaultFS of [" + options_.defaultFS.str() + "] is not supported";
     handler(Status::InvalidArgument(error_message.c_str()), nullptr);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
index e5a26fb449da4..b50134eda9536 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
@@ -16,7 +16,7 @@
 # limitations under the License.
 #
 
-list(APPEND rpc_object_items rpc_connection_impl.cc rpc_engine.cc namenode_tracker.cc request.cc sasl_protocol.cc sasl_engine.cc)
+list(APPEND rpc_object_items $<TARGET_OBJECTS:x_platform_obj> rpc_connection_impl.cc rpc_engine.cc namenode_tracker.cc request.cc sasl_protocol.cc sasl_engine.cc)
 if (CMAKE_USING_CYRUS_SASL)
   list(APPEND rpc_object_items cyrus_sasl_engine.cc)
 endif (CMAKE_USING_CYRUS_SASL)
@@ -25,7 +25,8 @@ if (CMAKE_USING_GSASL)
 endif (CMAKE_USING_GSASL)
 
 add_library(rpc_obj OBJECT ${rpc_object_items})
-
+target_include_directories(rpc_obj PRIVATE ../../lib)
 
 add_dependencies(rpc_obj proto)
 add_library(rpc $<TARGET_OBJECTS:rpc_obj>)
+target_include_directories(rpc PRIVATE ../../lib)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
index 6fc04f754f6cb..bc9adbff313d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
@@ -20,6 +20,7 @@
 #include "rpc_connection.h"
 #include "common/logging.h"
 #include "common/optional_wrapper.h"
+#include "x-platform/syscall.h"
 
 #include "sasl_engine.h"
 #include "sasl_protocol.h"
@@ -97,20 +98,17 @@ void SaslProtocol::Authenticate(std::function<void(const Status & status, const
     });
 } // authenticate() method
 
-AuthInfo::AuthMethod ParseMethod(const std::string & method)
-  {
-    if (0 == strcasecmp(method.c_str(), "SIMPLE")) {
-      return AuthInfo::kSimple;
-    }
-    else if (0 == strcasecmp(method.c_str(), "KERBEROS")) {
-      return AuthInfo::kKerberos;
-    }
-    else if (0 == strcasecmp(method.c_str(), "TOKEN")) {
-      return AuthInfo::kToken;
-    }
-    else {
-      return AuthInfo::kUnknownAuth;
-    }
+AuthInfo::AuthMethod ParseMethod(const std::string & method) {
+  if (XPlatform::Syscall::StringCompareIgnoreCase(method, "SIMPLE")) {
+    return AuthInfo::kSimple;
+  }
+  if (XPlatform::Syscall::StringCompareIgnoreCase(method, "KERBEROS")) {
+    return AuthInfo::kKerberos;
+  }
+  if (XPlatform::Syscall::StringCompareIgnoreCase(method, "TOKEN")) {
+    return AuthInfo::kToken;
+  }
+  return AuthInfo::kUnknownAuth;
 } // ParseMethod()
 
 // build_init_msg():
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
index 4f94ecbe31dfb..3a77f2dd6d026 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
@@ -71,6 +71,19 @@ class Syscall {
    */
   static void ClearBufferSafely(void* buffer, size_t sz_bytes);
 
+  /**
+   * Performs a case insensitive equality comparison for the two
+   * given strings {@link a} and {@link b}.
+   *
+   * @param a the first string parameter to compare.
+   * @param b the second string parameter to compare.
+   * @returns A boolean indicating whether to two strings are the
+   * same irrespective of their case. Returns true if they match,
+   * else false.
+   */
+  static bool StringCompareIgnoreCase(const std::string& a,
+                                      const std::string& b);
+
  private:
   static bool WriteToStdoutImpl(const char* message);
 };
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index 9821cc7110b2a..06e96a8732e37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -17,6 +17,7 @@
  */
 
 #include <fnmatch.h>
+#include <strings.h>
 #include <unistd.h>
 
 #include <cstring>
@@ -48,3 +49,8 @@ void XPlatform::Syscall::ClearBufferSafely(void* buffer,
     explicit_bzero(buffer, sz_bytes);
   }
 }
+
+bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
+                                                 const std::string& b) {
+  return strcasecmp(a.c_str(), b.c_str()) == 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
index 5a3423a99f196..2cd9e9d5157df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
@@ -20,6 +20,8 @@
 #include <WinBase.h>
 #include <Windows.h>
 
+#include <cstring>
+
 #include "syscall.h"
 
 #pragma comment(lib, "Shlwapi.lib")
@@ -57,3 +59,8 @@ void XPlatform::Syscall::ClearBufferSafely(void* buffer,
     SecureZeroMemory(buffer, sz_bytes);
   }
 }
+
+bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
+                                                 const std::string& b) {
+  return _stricmp(a.c_str(), b.c_str()) == 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
index d68b2afef9eea..7fa3971cf7edd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
@@ -67,4 +67,21 @@ TEST(XPlatformSyscall, ClearBufferSafelyNumbers) {
   for (const auto number : numbers) {
     EXPECT_EQ(number, 0);
   }
-}
\ No newline at end of file
+}
+
+TEST(XPlatformSyscall, StringCompareIgnoreCaseBasic) {
+  EXPECT_TRUE(XPlatform::Syscall::StringCompareIgnoreCase("aBcDeF", "AbCdEf"));
+  EXPECT_TRUE(XPlatform::Syscall::StringCompareIgnoreCase("a1B2c3D4e5F",
+                                                          "A1b2C3d4E5f"));
+  EXPECT_TRUE(XPlatform::Syscall::StringCompareIgnoreCase(
+      "a!1@B#2$c%3^D&4*e(5)F", "A!1@b#2$C%3^d&4*E(5)f"));
+  EXPECT_TRUE(XPlatform::Syscall::StringCompareIgnoreCase(
+      "a<!>1@B#2$c%3^D&4*e(5)F?:", "A<!>1@b#2$C%3^d&4*E(5)f?:"));
+  EXPECT_TRUE(XPlatform::Syscall::StringCompareIgnoreCase("12345", "12345"));
+  EXPECT_TRUE(XPlatform::Syscall::StringCompareIgnoreCase("", ""));
+}
+
+TEST(XPlatformSyscall, StringCompareIgnoreCaseNegative) {
+  EXPECT_FALSE(XPlatform::Syscall::StringCompareIgnoreCase("abcd", "abcde"));
+  EXPECT_FALSE(XPlatform::Syscall::StringCompareIgnoreCase("12345", "abcde"));
+}

From 6a9a5ae5fdc80d53185938a3c6c9e902b5ba2244 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 11 Apr 2021 01:30:49 +0900
Subject: [PATCH 19/43] Bump y18n (#2843)

Bumps [y18n](https://github.com/yargs/y18n) from 3.2.1 to 3.2.2.
- [Release notes](https://github.com/yargs/y18n/releases)
- [Changelog](https://github.com/yargs/y18n/blob/master/CHANGELOG.md)
- [Commits](https://github.com/yargs/y18n/commits)

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
---
 .../hadoop-yarn-ui/src/main/webapp/yarn.lock  | 28 ++++++-------------
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
index bbafc667a30ef..1d36e689da053 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
@@ -79,11 +79,6 @@ amdefine@>=0.0.4:
   resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5"
   integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=
 
-ansi-regex@*, ansi-regex@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
-  integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
-
 ansi-regex@^0.2.0, ansi-regex@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-0.2.1.tgz#0d8e946967a3d8143f93e24e298525fc1b2235f9"
@@ -6163,16 +6158,16 @@ spdx-expression-parse@^3.0.0:
     spdx-exceptions "^2.1.0"
     spdx-license-ids "^3.0.0"
 
-spdx-license-ids@*, spdx-license-ids@^3.0.0:
-  version "3.0.5"
-  resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz#3694b5804567a458d3c8045842a6358632f62654"
-  integrity sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==
-
 spdx-license-ids@^1.0.0:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz#c9df7a3424594ade6bd11900d596696dc06bac57"
   integrity sha1-yd96NCRZSt5r0RkA1ZZpbcBrrFc=
 
+spdx-license-ids@^3.0.0:
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz#3694b5804567a458d3c8045842a6358632f62654"
+  integrity sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==
+
 spdx@~0.4.1:
   version "0.4.3"
   resolved "https://registry.yarnpkg.com/spdx/-/spdx-0.4.3.tgz#ab373c3fcf7b84ffd8fdeb0592d24ff0d14812e4"
@@ -6276,13 +6271,6 @@ stringstream@~0.0.4:
   resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.6.tgz#7880225b0d4ad10e30927d167a1d6f2fd3b33a72"
   integrity sha512-87GEBAkegbBcweToUrdzf3eLhWNg06FJTebl4BVJz/JgWy8CvEr9dRtX5qWphiynMSQlxxi+QqN0z5T32SLlhA==
 
-strip-ansi@*:
-  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
-  integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==
-  dependencies:
-    ansi-regex "^5.0.0"
-
 strip-ansi@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-0.3.0.tgz#25f48ea22ca79187f3174a4db8759347bb126220"
@@ -6906,9 +6894,9 @@ xtend@^4.0.0:
   integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
 
 y18n@^3.2.0, y18n@^3.2.1:
-  version "3.2.1"
-  resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41"
-  integrity sha1-bRX7qITAhnnA136I53WegR4H+kE=
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.2.tgz#85c901bd6470ce71fc4bb723ad209b70f7f28696"
+  integrity sha512-uGZHXkHnhF0XeeAPgnKfPv1bgKAYyVvmNL1xlKsPYZPaIHxGti2hHqvOCQv71XMsLxu1QjergkqogUnms5D3YQ==
 
 yallist@^2.1.2:
   version "2.1.2"

From 2bd810a5075e6b9c15fde22e43ac9bcf2a6c22f9 Mon Sep 17 00:00:00 2001
From: Akira Ajisaka <aajisaka@apache.org>
Date: Mon, 12 Apr 2021 12:53:58 +0900
Subject: [PATCH 20/43] HADOOP-17608. Fix TestKMS failure (#2880)

Reviewed-by: Masatake Iwasaki <iwasakims@apache.org>
---
 .../hadoop/crypto/key/kms/server/TestKMS.java | 52 +------------------
 1 file changed, 1 insertion(+), 51 deletions(-)

diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 3d59e6f5be7b7..a0a58ff3567f5 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.crypto.key.kms.server;
 
-import java.util.function.Supplier;
 import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
 import org.apache.curator.test.TestingServer;
 import org.apache.hadoop.conf.Configuration;
@@ -92,7 +91,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -113,9 +111,6 @@
 public class TestKMS {
   private static final Logger LOG = LoggerFactory.getLogger(TestKMS.class);
 
-  private static final String SSL_RELOADER_THREAD_NAME =
-      "Truststore reloader thread";
-
   private SSLFactory sslFactory;
 
   // Keep track of all key providers created during a test case, so they can be
@@ -540,34 +535,6 @@ public Void call() throws Exception {
             url.getProtocol().equals("https"));
         final URI uri = createKMSUri(getKMSUrl());
 
-        if (ssl) {
-          KeyProvider testKp = createProvider(uri, conf);
-          ThreadGroup threadGroup = Thread.currentThread().getThreadGroup();
-          while (threadGroup.getParent() != null) {
-            threadGroup = threadGroup.getParent();
-          }
-          Thread[] threads = new Thread[threadGroup.activeCount()];
-          threadGroup.enumerate(threads);
-          Thread reloaderThread = null;
-          for (Thread thread : threads) {
-            if ((thread.getName() != null)
-                && (thread.getName().contains(SSL_RELOADER_THREAD_NAME))) {
-              reloaderThread = thread;
-            }
-          }
-          Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive());
-          // Explicitly close the provider so we can verify the internal thread
-          // is shutdown
-          testKp.close();
-          boolean reloaderStillAlive = true;
-          for (int i = 0; i < 10; i++) {
-            reloaderStillAlive = reloaderThread.isAlive();
-            if (!reloaderStillAlive) break;
-            Thread.sleep(1000);
-          }
-          Assert.assertFalse("Reloader is still alive", reloaderStillAlive);
-        }
-
         if (kerberos) {
           for (String user : new String[]{"client", "client/host"}) {
             doAs(user, new PrivilegedExceptionAction<Void>() {
@@ -2363,8 +2330,7 @@ public Void run() throws Exception {
                   return null;
                 }
               });
-              // Close the client provider. We will verify all providers'
-              // Truststore reloader threads are closed later.
+              // Close the client provider.
               kp.close();
               return null;
             } finally {
@@ -2375,22 +2341,6 @@ public Void run() throws Exception {
         return null;
       }
     });
-
-    // verify that providers created by KMSTokenRenewer are closed.
-    if (ssl) {
-      GenericTestUtils.waitFor(new Supplier<Boolean>() {
-        @Override
-        public Boolean get() {
-          final Set<Thread> threadSet = Thread.getAllStackTraces().keySet();
-          for (Thread t : threadSet) {
-            if (t.getName().contains(SSL_RELOADER_THREAD_NAME)) {
-              return false;
-            }
-          }
-          return true;
-        }
-      }, 1000, 10000);
-    }
   }
 
   @Test

From cb3ed32fe0d937980999d590deb35ad97d40f9e6 Mon Sep 17 00:00:00 2001
From: lfengnan <lfengnan@uber.com>
Date: Mon, 12 Apr 2021 12:42:33 -0700
Subject: [PATCH 21/43] HDFS-15423 RBF: WebHDFS create shouldn't choose DN from
 all sub-clusters (#2605)

---
 .../router/RouterWebHdfsMethods.java          |  47 +++++-
 .../router/web/RouterWebHDFSContract.java     |   2 +
 .../federation/MiniRouterDFSCluster.java      |  19 +--
 .../router/TestRouterWebHdfsMethods.java      | 147 ++++++++++++++++++
 4 files changed, 198 insertions(+), 17 deletions(-)
 create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
index f6ac70c368a52..afc4a3d8fac21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
@@ -93,6 +93,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -104,6 +105,8 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * WebHDFS Router implementation. This is an extension of
@@ -453,21 +456,33 @@ private DatanodeInfo chooseDatanode(final Router router,
       final String path, final HttpOpParam.Op op, final long openOffset,
       final String excludeDatanodes) throws IOException {
     final RouterRpcServer rpcServer = getRPCServer(router);
-    DatanodeInfo[] dns = null;
+    DatanodeInfo[] dns = {};
+    String resolvedNs = "";
     try {
       dns = rpcServer.getCachedDatanodeReport(DatanodeReportType.LIVE);
     } catch (IOException e) {
       LOG.error("Cannot get the datanodes from the RPC server", e);
     }
 
+    if (op == PutOpParam.Op.CREATE) {
+      try {
+        resolvedNs = rpcServer.getCreateLocation(path).getNameserviceId();
+      } catch (IOException e) {
+        LOG.error("Cannot get the name service " +
+            "to create file for path {} ", path, e);
+      }
+    }
+
     HashSet<Node> excludes = new HashSet<Node>();
-    if (excludeDatanodes != null) {
-      Collection<String> collection =
-          getTrimmedStringCollection(excludeDatanodes);
-      for (DatanodeInfo dn : dns) {
-        if (collection.contains(dn.getName())) {
-          excludes.add(dn);
-        }
+    Collection<String> collection =
+        getTrimmedStringCollection(excludeDatanodes);
+    for (DatanodeInfo dn : dns) {
+      String ns = getNsFromDataNodeNetworkLocation(dn.getNetworkLocation());
+      if (collection.contains(dn.getName())) {
+        excludes.add(dn);
+      } else if (op == PutOpParam.Op.CREATE && !ns.equals(resolvedNs)) {
+        // for CREATE, the dest dn should be in the resolved ns
+        excludes.add(dn);
       }
     }
 
@@ -502,6 +517,22 @@ private DatanodeInfo chooseDatanode(final Router router,
     return getRandomDatanode(dns, excludes);
   }
 
+  /**
+   * Get the nameservice info from datanode network location.
+   * @param location network location with format `/ns0/rack1`
+   * @return nameservice this datanode is in
+   */
+  @VisibleForTesting
+  public static String getNsFromDataNodeNetworkLocation(String location) {
+    // network location should be in the format of /ns/rack
+    Pattern pattern = Pattern.compile("^/([^/]*)/");
+    Matcher matcher = pattern.matcher(location);
+    if (matcher.find()) {
+      return matcher.group(1);
+    }
+    return "";
+  }
+
   /**
    * Get a random Datanode from a subcluster.
    * @param dns Nodes to be chosen from.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
index 1d308073290d4..6b90faecc78f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
@@ -64,6 +64,8 @@ public static void createCluster(Configuration conf) throws IOException {
       conf.addResource(CONTRACT_WEBHDFS_XML);
 
       cluster = new MiniRouterDFSCluster(true, 2, conf);
+      cluster.setIndependentDNs();
+      cluster.setNumDatanodesPerNameservice(3);
 
       // Start NNs and DNs and wait until ready
       cluster.startCluster(conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
index 896d08f2c49b6..8a7a03e018b95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
@@ -774,6 +774,15 @@ public void startCluster(Configuration overrideConf) {
       }
       topology.setFederation(true);
 
+      // Generate conf for namenodes and datanodes
+      String ns0 = nameservices.get(0);
+      Configuration nnConf = generateNamenodeConfiguration(ns0);
+      if (overrideConf != null) {
+        nnConf.addResource(overrideConf);
+        // Router also uses this configurations as initial values.
+        routerConf = new Configuration(overrideConf);
+      }
+
       // Set independent DNs across subclusters
       int numDNs = nameservices.size() * numDatanodesPerNameservice;
       Configuration[] dnConfs = null;
@@ -781,7 +790,7 @@ public void startCluster(Configuration overrideConf) {
         dnConfs = new Configuration[numDNs];
         int dnId = 0;
         for (String nsId : nameservices) {
-          Configuration subclusterConf = new Configuration();
+          Configuration subclusterConf = new Configuration(nnConf);
           subclusterConf.set(DFS_INTERNAL_NAMESERVICES_KEY, nsId);
           for (int i = 0; i < numDatanodesPerNameservice; i++) {
             dnConfs[dnId] = subclusterConf;
@@ -791,14 +800,6 @@ public void startCluster(Configuration overrideConf) {
       }
 
       // Start mini DFS cluster
-      String ns0 = nameservices.get(0);
-      Configuration nnConf = generateNamenodeConfiguration(ns0);
-      if (overrideConf != null) {
-        nnConf.addResource(overrideConf);
-        // Router also uses this configurations as initial values.
-        routerConf = new Configuration(overrideConf);
-      }
-
       cluster = new MiniDFSCluster.Builder(nnConf)
           .numDataNodes(numDNs)
           .nnTopology(topology)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
new file mode 100644
index 0000000000000..7028928041452
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createMountTableEntry;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.FileNotFoundException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Collections;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test suite for Router Web Hdfs methods.
+ */
+public class TestRouterWebHdfsMethods {
+  static final Logger LOG =
+      LoggerFactory.getLogger(TestRouterWebHdfsMethods.class);
+
+  private static StateStoreDFSCluster cluster;
+  private static RouterContext router;
+  private static String httpUri;
+
+  @BeforeClass
+  public static void globalSetUp() throws Exception {
+    cluster = new StateStoreDFSCluster(false, 2);
+    Configuration conf = new RouterConfigBuilder()
+        .stateStore()
+        .rpc()
+        .http()
+        .admin()
+        .build();
+    cluster.addRouterOverrides(conf);
+    cluster.startCluster();
+    cluster.startRouters();
+    cluster.waitClusterUp();
+    router = cluster.getRandomRouter();
+    httpUri = "http://"+router.getHttpAddress();
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @Test
+  public void testWebHdfsCreate() throws Exception {
+    // the file is created at default ns (ns0)
+    String path = "/tmp/file";
+    URL url = new URL(getUri(path));
+    LOG.info("URL: {}", url);
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
+    verifyFile("ns0", path, true);
+    verifyFile("ns1", path, false);
+    conn.disconnect();
+  }
+
+  @Test
+  public void testWebHdfsCreateWithMounts() throws Exception {
+    // the file is created at mounted ns (ns1)
+    String mountPoint = "/tmp-ns1";
+    String path = "/tmp-ns1/file";
+    createMountTableEntry(
+        router.getRouter(), mountPoint,
+        DestinationOrder.RANDOM, Collections.singletonList("ns1"));
+    URL url = new URL(getUri(path));
+    LOG.info("URL: {}", url);
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
+    verifyFile("ns1", path, true);
+    verifyFile("ns0", path, false);
+    conn.disconnect();
+  }
+
+  private String getUri(String path) {
+    final String user = System.getProperty("user.name");
+    final StringBuilder uri = new StringBuilder(httpUri);
+    uri.append("/webhdfs/v1").
+        append(path).
+        append("?op=CREATE").
+        append("&user.name=" + user);
+    return uri.toString();
+  }
+
+  private void verifyFile(String ns, String path, boolean shouldExist)
+      throws Exception {
+    FileSystem fs = cluster.getNamenode(ns, null).getFileSystem();
+    try {
+      fs.getFileStatus(new Path(path));
+      if (!shouldExist) {
+        fail(path + " should not exist in ns " + ns);
+      }
+    } catch (FileNotFoundException e) {
+      if (shouldExist) {
+        fail(path + " should exist in ns " + ns);
+      }
+    }
+  }
+
+  @Test
+  public void testGetNsFromDataNodeNetworkLocation() {
+    assertEquals("ns0", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("/ns0/rack-info1"));
+    assertEquals("ns0", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("/ns0/row1/rack-info1"));
+    assertEquals("", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("/row0"));
+    assertEquals("", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("whatever-rack-info1"));
+  }
+}

From c1fde4fe94f268c6d5515b421ac47345dca8163d Mon Sep 17 00:00:00 2001
From: billierinaldi <billie@apache.org>
Date: Mon, 12 Apr 2021 19:47:59 -0400
Subject: [PATCH 22/43] HADOOP-16948. Support infinite lease dirs. (#1925)

* HADOOP-16948. Support single writer dirs.

* HADOOP-16948. Fix findbugs and checkstyle problems.

* HADOOP-16948. Fix remaining checkstyle problems.

* HADOOP-16948. Add DurationInfo, retry policy for acquiring lease, and javadocs

* HADOOP-16948. Convert ABFS client to use an executor for lease ops

* HADOOP-16948. Fix ABFS lease test for non-HNS

* HADOOP-16948. Fix checkstyle and javadoc

* HADOOP-16948. Address review comments

* HADOOP-16948. Use daemon threads for ABFS lease ops

* HADOOP-16948. Make lease duration configurable

* HADOOP-16948. Add error messages to test assertions

* HADOOP-16948. Remove extra isSingleWriterKey call

* HADOOP-16948. Use only infinite lease duration due to cost of renewal ops

* HADOOP-16948. Remove acquire/renew/release lease methods

* HADOOP-16948. Rename single writer dirs to infinite lease dirs

* HADOOP-16948. Fix checkstyle

* HADOOP-16948. Wait for acquire lease future

* HADOOP-16948. Add unit test for acquire lease failure
---
 .../hadoop/fs/azurebfs/AbfsConfiguration.java |  35 ++
 .../fs/azurebfs/AzureBlobFileSystem.java      |  21 ++
 .../fs/azurebfs/AzureBlobFileSystemStore.java |  90 ++++-
 .../azurebfs/constants/AbfsHttpConstants.java |   5 +
 .../azurebfs/constants/ConfigurationKeys.java |   9 +
 .../constants/FileSystemConfigurations.java   |   7 +
 .../constants/HttpHeaderConfigurations.java   |   5 +
 .../ConfigurationValidationAnnotations.java   |  16 +
 .../AzureBlobFileSystemException.java         |   4 +
 .../services/AppendRequestParameters.java     |   8 +-
 .../IntegerConfigurationBasicValidator.java   |  13 +-
 .../fs/azurebfs/services/AbfsClient.java      | 131 ++++++-
 .../fs/azurebfs/services/AbfsErrors.java      |  53 +++
 .../fs/azurebfs/services/AbfsLease.java       | 188 ++++++++++
 .../azurebfs/services/AbfsOutputStream.java   |  37 +-
 .../services/AbfsOutputStreamContext.java     |  18 +
 .../azurebfs/services/AbfsRestOperation.java  |   1 +
 .../services/AbfsRestOperationType.java       |   3 +-
 .../hadoop-azure/src/site/markdown/abfs.md    |  16 +
 .../ITestAzureBlobFileSystemLease.java        | 336 ++++++++++++++++++
 .../TestConfigurationValidators.java          |  29 +-
 .../services/TestAbfsOutputStream.java        |  49 +--
 22 files changed, 1032 insertions(+), 42 deletions(-)
 create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java
 create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java
 create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index f36cc7d5bfde0..0a8224aaaeb58 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
 import org.apache.hadoop.fs.azurebfs.constants.AuthConfigurations;
 import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation;
+import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerWithOutlierConfigurationValidatorAnnotation;
 import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation;
 import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
 import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.Base64StringConfigurationValidatorAnnotation;
@@ -208,6 +209,15 @@ public class AbfsConfiguration{
       DefaultValue = DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES)
   private String azureAppendBlobDirs;
 
+  @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_INFINITE_LEASE_KEY,
+      DefaultValue = DEFAULT_FS_AZURE_INFINITE_LEASE_DIRECTORIES)
+  private String azureInfiniteLeaseDirs;
+
+  @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_LEASE_THREADS,
+      MinValue = MIN_LEASE_THREADS,
+      DefaultValue = DEFAULT_LEASE_THREADS)
+  private int numLeaseThreads;
+
   @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
       DefaultValue = DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION)
   private boolean createRemoteFileSystemDuringInitialization;
@@ -296,6 +306,8 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName)
       field.setAccessible(true);
       if (field.isAnnotationPresent(IntegerConfigurationValidatorAnnotation.class)) {
         field.set(this, validateInt(field));
+      } else if (field.isAnnotationPresent(IntegerWithOutlierConfigurationValidatorAnnotation.class)) {
+        field.set(this, validateIntWithOutlier(field));
       } else if (field.isAnnotationPresent(LongConfigurationValidatorAnnotation.class)) {
         field.set(this, validateLong(field));
       } else if (field.isAnnotationPresent(StringConfigurationValidatorAnnotation.class)) {
@@ -634,6 +646,14 @@ public String getAppendBlobDirs() {
     return this.azureAppendBlobDirs;
   }
 
+  public String getAzureInfiniteLeaseDirs() {
+    return this.azureInfiniteLeaseDirs;
+  }
+
+  public int getNumLeaseThreads() {
+    return this.numLeaseThreads;
+  }
+
   public boolean getCreateRemoteFileSystemDuringInitialization() {
     // we do not support creating the filesystem when AuthType is SAS
     return this.createRemoteFileSystemDuringInitialization
@@ -843,6 +863,21 @@ int validateInt(Field field) throws IllegalAccessException, InvalidConfiguration
         validator.ThrowIfInvalid()).validate(value);
   }
 
+  int validateIntWithOutlier(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
+    IntegerWithOutlierConfigurationValidatorAnnotation validator =
+        field.getAnnotation(IntegerWithOutlierConfigurationValidatorAnnotation.class);
+    String value = get(validator.ConfigurationKey());
+
+    // validate
+    return new IntegerConfigurationBasicValidator(
+        validator.OutlierValue(),
+        validator.MinValue(),
+        validator.MaxValue(),
+        validator.DefaultValue(),
+        validator.ConfigurationKey(),
+        validator.ThrowIfInvalid()).validate(value);
+  }
+
   long validateLong(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
     LongConfigurationValidatorAnnotation validator = field.getAnnotation(LongConfigurationValidatorAnnotation.class);
     String value = rawConfig.get(validator.ConfigurationKey());
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index d8a2ed7bcd542..30108ed1e2fb3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -87,6 +87,7 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.functional.RemoteIterators;
+import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.LambdaUtils;
 import org.apache.hadoop.util.Progressable;
 
@@ -505,6 +506,26 @@ public FileStatus getFileStatus(final Path f) throws IOException {
     }
   }
 
+  /**
+   * Break the current lease on an ABFS file if it exists. A lease that is broken cannot be
+   * renewed. A new lease may be obtained on the file immediately.
+   *
+   * @param f file name
+   * @throws IOException on any exception while breaking the lease
+   */
+  public void breakLease(final Path f) throws IOException {
+    LOG.debug("AzureBlobFileSystem.breakLease path: {}", f);
+
+    Path qualifiedPath = makeQualified(f);
+
+    try (DurationInfo ignored = new DurationInfo(LOG, false, "Break lease for %s",
+        qualifiedPath)) {
+      abfsStore.breakLease(qualifiedPath);
+    } catch(AzureBlobFileSystemException ex) {
+      checkException(f, ex);
+    }
+  }
+
   /**
    * Qualify a path to one which uses this FileSystem and, if relative,
    * made absolute.
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 75419c26dd214..fa7e12bc80e28 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -39,6 +39,7 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -48,10 +49,14 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.WeakHashMap;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -100,6 +105,7 @@
 import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
 import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
+import org.apache.hadoop.fs.azurebfs.services.AbfsLease;
 import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials;
 import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker;
 import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo;
@@ -145,8 +151,11 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
   private static final String XMS_PROPERTIES_ENCODING = "ISO-8859-1";
   private static final int GET_SET_AGGREGATE_COUNT = 2;
 
+  private final Map<AbfsLease, Object> leaseRefs;
+
   private final AbfsConfiguration abfsConfiguration;
   private final Set<String> azureAtomicRenameDirSet;
+  private Set<String> azureInfiniteLeaseDirSet;
   private Trilean isNamespaceEnabled;
   private final AuthType authType;
   private final UserGroupInformation userGroupInformation;
@@ -167,6 +176,8 @@ public AzureBlobFileSystemStore(URI uri, boolean isSecureScheme,
     final String fileSystemName = authorityParts[0];
     final String accountName = authorityParts[1];
 
+    leaseRefs = Collections.synchronizedMap(new WeakHashMap<>());
+
     try {
       this.abfsConfiguration = new AbfsConfiguration(configuration, accountName);
     } catch (IllegalAccessException exception) {
@@ -195,6 +206,7 @@ public AzureBlobFileSystemStore(URI uri, boolean isSecureScheme,
 
     this.azureAtomicRenameDirSet = new HashSet<>(Arrays.asList(
         abfsConfiguration.getAzureAtomicRenameDirs().split(AbfsHttpConstants.COMMA)));
+    updateInfiniteLeaseDirs();
     this.authType = abfsConfiguration.getAuthType(accountName);
     boolean usingOauth = (authType == AuthType.OAuth);
     boolean useHttps = (usingOauth || abfsConfiguration.isHttpsAlwaysUsed()) ? true : isSecureScheme;
@@ -246,7 +258,24 @@ public String getPrimaryGroup() {
 
   @Override
   public void close() throws IOException {
-    IOUtils.cleanupWithLogger(LOG, client);
+    List<ListenableFuture<?>> futures = new ArrayList<>();
+    for (AbfsLease lease : leaseRefs.keySet()) {
+      if (lease == null) {
+        continue;
+      }
+      ListenableFuture<?> future = client.submit(() -> lease.free());
+      futures.add(future);
+    }
+    try {
+      Futures.allAsList(futures).get();
+    } catch (InterruptedException e) {
+      LOG.error("Interrupted freeing leases", e);
+      Thread.currentThread().interrupt();
+    } catch (ExecutionException e) {
+      LOG.error("Error freeing leases", e);
+    } finally {
+      IOUtils.cleanupWithLogger(LOG, client);
+    }
   }
 
   byte[] encodeAttribute(String value) throws UnsupportedEncodingException {
@@ -496,12 +525,14 @@ public OutputStream createFile(final Path path,
       }
       perfInfo.registerResult(op.getResult()).registerSuccess(true);
 
+      AbfsLease lease = maybeCreateLease(relativePath);
+
       return new AbfsOutputStream(
           client,
           statistics,
           relativePath,
           0,
-          populateAbfsOutputStreamContext(isAppendBlob));
+          populateAbfsOutputStreamContext(isAppendBlob, lease));
     }
   }
 
@@ -573,7 +604,8 @@ private AbfsRestOperation conditionalCreateOverwriteFile(final String relativePa
     return op;
   }
 
-  private AbfsOutputStreamContext populateAbfsOutputStreamContext(boolean isAppendBlob) {
+  private AbfsOutputStreamContext populateAbfsOutputStreamContext(boolean isAppendBlob,
+      AbfsLease lease) {
     int bufferSize = abfsConfiguration.getWriteBufferSize();
     if (isAppendBlob && bufferSize > FileSystemConfigurations.APPENDBLOB_MAX_WRITE_BUFFER_SIZE) {
       bufferSize = FileSystemConfigurations.APPENDBLOB_MAX_WRITE_BUFFER_SIZE;
@@ -587,6 +619,7 @@ private AbfsOutputStreamContext populateAbfsOutputStreamContext(boolean isAppend
             .withAppendBlob(isAppendBlob)
             .withWriteMaxConcurrentRequestCount(abfsConfiguration.getWriteMaxConcurrentRequestCount())
             .withMaxWriteRequestsToQueue(abfsConfiguration.getMaxWriteRequestsToQueue())
+            .withLease(lease)
             .build();
   }
 
@@ -705,15 +738,29 @@ public OutputStream openFileForWrite(final Path path, final FileSystem.Statistic
         isAppendBlob = true;
       }
 
+      AbfsLease lease = maybeCreateLease(relativePath);
+
       return new AbfsOutputStream(
           client,
           statistics,
           relativePath,
           offset,
-          populateAbfsOutputStreamContext(isAppendBlob));
+          populateAbfsOutputStreamContext(isAppendBlob, lease));
     }
   }
 
+  /**
+   * Break any current lease on an ABFS file.
+   *
+   * @param path file name
+   * @throws AzureBlobFileSystemException on any exception while breaking the lease
+   */
+  public void breakLease(final Path path) throws AzureBlobFileSystemException {
+    LOG.debug("lease path: {}", path);
+
+    client.breakLease(getRelativePath(path));
+  }
+
   public void rename(final Path source, final Path destination) throws
           AzureBlobFileSystemException {
     final Instant startAggregate = abfsPerfTracker.getLatencyInstant();
@@ -1347,6 +1394,13 @@ public boolean isAtomicRenameKey(String key) {
     return isKeyForDirectorySet(key, azureAtomicRenameDirSet);
   }
 
+  public boolean isInfiniteLeaseKey(String key) {
+    if (azureInfiniteLeaseDirSet.isEmpty()) {
+      return false;
+    }
+    return isKeyForDirectorySet(key, azureInfiniteLeaseDirSet);
+  }
+
   /**
    * A on-off operation to initialize AbfsClient for AzureBlobFileSystem
    * Operations.
@@ -1636,4 +1690,32 @@ void setNamespaceEnabled(Trilean isNamespaceEnabled){
     this.isNamespaceEnabled = isNamespaceEnabled;
   }
 
+  private void updateInfiniteLeaseDirs() {
+    this.azureInfiniteLeaseDirSet = new HashSet<>(Arrays.asList(
+        abfsConfiguration.getAzureInfiniteLeaseDirs().split(AbfsHttpConstants.COMMA)));
+    // remove the empty string, since isKeyForDirectory returns true for empty strings
+    // and we don't want to default to enabling infinite lease dirs
+    this.azureInfiniteLeaseDirSet.remove("");
+  }
+
+  private AbfsLease maybeCreateLease(String relativePath)
+      throws AzureBlobFileSystemException {
+    boolean enableInfiniteLease = isInfiniteLeaseKey(relativePath);
+    if (!enableInfiniteLease) {
+      return null;
+    }
+    AbfsLease lease = new AbfsLease(client, relativePath);
+    leaseRefs.put(lease, null);
+    return lease;
+  }
+
+  @VisibleForTesting
+  boolean areLeasesFreed() {
+    for (AbfsLease lease : leaseRefs.keySet()) {
+      if (lease != null && !lease.isFreed()) {
+        return false;
+      }
+    }
+    return true;
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java
index 184657e7d66ad..5cf7ec565b59e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java
@@ -39,6 +39,11 @@ public final class AbfsHttpConstants {
   public static final String GET_ACCESS_CONTROL = "getAccessControl";
   public static final String CHECK_ACCESS = "checkAccess";
   public static final String GET_STATUS = "getStatus";
+  public static final String ACQUIRE_LEASE_ACTION = "acquire";
+  public static final String BREAK_LEASE_ACTION = "break";
+  public static final String RELEASE_LEASE_ACTION = "release";
+  public static final String RENEW_LEASE_ACTION = "renew";
+  public static final String DEFAULT_LEASE_BREAK_PERIOD = "0";
   public static final String DEFAULT_TIMEOUT = "90";
   public static final String APPEND_BLOB_TYPE = "appendblob";
   public static final String TOKEN_VERSION = "2";
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 02b143cd61bd3..4fe1d1c276db5 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -87,6 +87,15 @@ public final class ConfigurationKeys {
   /** Provides a config to provide comma separated path prefixes on which Appendblob based files are created
    *  Default is empty. **/
   public static final String FS_AZURE_APPEND_BLOB_KEY = "fs.azure.appendblob.directories";
+  /** Provides a config to provide comma separated path prefixes which support infinite leases.
+   *  Files under these paths will be leased when created or opened for writing and the lease will
+   *  be released when the file is closed. The lease may be broken with the breakLease method on
+   *  AzureBlobFileSystem. Default is empty.
+   * **/
+  public static final String FS_AZURE_INFINITE_LEASE_KEY = "fs.azure.infinite-lease.directories";
+  /** Provides a number of threads to use for lease operations for infinite lease directories.
+   *  Must be set to a minimum of 1 if infinite lease directories are to be used. Default is 0. **/
+  public static final String FS_AZURE_LEASE_THREADS = "fs.azure.lease.threads";
   public static final String FS_AZURE_READ_AHEAD_QUEUE_DEPTH = "fs.azure.readaheadqueue.depth";
   public static final String FS_AZURE_ALWAYS_READ_BUFFER_SIZE = "fs.azure.read.alwaysReadBufferSize";
   public static final String FS_AZURE_READ_AHEAD_BLOCK_SIZE = "fs.azure.read.readahead.blocksize";
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index d90f525712af7..040b18ae4c281 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -78,6 +78,13 @@ public final class FileSystemConfigurations {
   public static final boolean DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE = true;
   public static final boolean DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE = true;
   public static final String DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES = "";
+  public static final String DEFAULT_FS_AZURE_INFINITE_LEASE_DIRECTORIES = "";
+  public static final int DEFAULT_LEASE_THREADS = 0;
+  public static final int MIN_LEASE_THREADS = 0;
+  public static final int DEFAULT_LEASE_DURATION = -1;
+  public static final int INFINITE_LEASE_DURATION = -1;
+  public static final int MIN_LEASE_DURATION = 15;
+  public static final int MAX_LEASE_DURATION = 60;
 
   public static final int DEFAULT_READ_AHEAD_QUEUE_DEPTH = -1;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
index 27ddcee695aaa..232553844fcf3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
@@ -60,6 +60,11 @@ public final class HttpHeaderConfigurations {
   public static final String X_MS_UMASK = "x-ms-umask";
   public static final String X_MS_NAMESPACE_ENABLED = "x-ms-namespace-enabled";
   public static final String X_MS_ABFS_CLIENT_LATENCY = "x-ms-abfs-client-latency";
+  public static final String X_MS_LEASE_ACTION = "x-ms-lease-action";
+  public static final String X_MS_LEASE_DURATION = "x-ms-lease-duration";
+  public static final String X_MS_LEASE_ID = "x-ms-lease-id";
+  public static final String X_MS_PROPOSED_LEASE_ID = "x-ms-proposed-lease-id";
+  public static final String X_MS_LEASE_BREAK_PERIOD = "x-ms-lease-break-period";
 
   private HttpHeaderConfigurations() {}
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java
index 82c571a3b03b3..9fbe5a22cdf77 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/annotations/ConfigurationValidationAnnotations.java
@@ -46,6 +46,22 @@ public class ConfigurationValidationAnnotations {
     boolean ThrowIfInvalid() default false;
   }
 
+  @Target({ ElementType.FIELD })
+  @Retention(RetentionPolicy.RUNTIME)
+  public @interface IntegerWithOutlierConfigurationValidatorAnnotation {
+    String ConfigurationKey();
+
+    int MaxValue() default Integer.MAX_VALUE;
+
+    int MinValue() default Integer.MIN_VALUE;
+
+    int OutlierValue() default Integer.MIN_VALUE;
+
+    int DefaultValue();
+
+    boolean ThrowIfInvalid() default false;
+  }
+
   /**
    * Describes the requirements when validating the annotated long field.
    */
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java
index 9b1bead886e6c..d829c5ac6779c 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AzureBlobFileSystemException.java
@@ -37,6 +37,10 @@ public AzureBlobFileSystemException(final String message, final Exception innerE
     super(message, innerException);
   }
 
+  public AzureBlobFileSystemException(final String message, final Throwable innerThrowable) {
+    super(message, innerThrowable);
+  }
+
   @Override
   public String toString() {
     if (this.getMessage() == null && this.getCause() == null) {
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java
index fb4d29f87949a..7369bfaf56422 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java
@@ -33,17 +33,20 @@ public enum Mode {
   private final int length;
   private final Mode mode;
   private final boolean isAppendBlob;
+  private final String leaseId;
 
   public AppendRequestParameters(final long position,
       final int offset,
       final int length,
       final Mode mode,
-      final boolean isAppendBlob) {
+      final boolean isAppendBlob,
+      final String leaseId) {
     this.position = position;
     this.offset = offset;
     this.length = length;
     this.mode = mode;
     this.isAppendBlob = isAppendBlob;
+    this.leaseId = leaseId;
   }
 
   public long getPosition() {
@@ -66,4 +69,7 @@ public boolean isAppendBlob() {
     return this.isAppendBlob;
   }
 
+  public String getLeaseId() {
+    return this.leaseId;
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/diagnostics/IntegerConfigurationBasicValidator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/diagnostics/IntegerConfigurationBasicValidator.java
index 26c7d2f0ac19c..9d4beb74bbe3f 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/diagnostics/IntegerConfigurationBasicValidator.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/diagnostics/IntegerConfigurationBasicValidator.java
@@ -31,11 +31,18 @@
 public class IntegerConfigurationBasicValidator extends ConfigurationBasicValidator<Integer> implements ConfigurationValidator {
   private final int min;
   private final int max;
+  private final int outlier;
 
   public IntegerConfigurationBasicValidator(final int min, final int max, final int defaultVal, final String configKey, final boolean throwIfInvalid) {
+    this(min, min, max, defaultVal, configKey, throwIfInvalid);
+  }
+
+  public IntegerConfigurationBasicValidator(final int outlier, final int min, final int max,
+      final int defaultVal, final String configKey, final boolean throwIfInvalid) {
     super(configKey, defaultVal, throwIfInvalid);
     this.min = min;
     this.max = max;
+    this.outlier = outlier;
   }
 
   public Integer validate(final String configValue) throws InvalidConfigurationValueException {
@@ -47,10 +54,14 @@ public Integer validate(final String configValue) throws InvalidConfigurationVal
     try {
       result = Integer.parseInt(configValue);
       // throw an exception if a 'within bounds' value is missing
-      if (getThrowIfInvalid() && (result < this.min || result > this.max)) {
+      if (getThrowIfInvalid() && (result != outlier) && (result < this.min || result > this.max)) {
         throw new InvalidConfigurationValueException(getConfigKey());
       }
 
+      if (result == outlier) {
+        return result;
+      }
+
       // set the value to the nearest bound if it's out of bounds
       if (result < this.min) {
         return this.min;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index 92b24f0dda212..7c8a2112bfa46 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -29,16 +29,27 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
-import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
-import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
-import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
-import org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableScheduledFuture;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningScheduledExecutorService;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
+import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
+import org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException;
@@ -49,6 +60,8 @@
 import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
 import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
 
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT;
@@ -76,6 +89,8 @@ public class AbfsClient implements Closeable {
   private SASTokenProvider sasTokenProvider;
   private final AbfsCounters abfsCounters;
 
+  private final ListeningScheduledExecutorService executorService;
+
   private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials,
                     final AbfsConfiguration abfsConfiguration,
                     final AbfsClientContext abfsClientContext) {
@@ -106,6 +121,11 @@ private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCreden
     this.userAgent = initializeUserAgent(abfsConfiguration, sslProviderName);
     this.abfsPerfTracker = abfsClientContext.getAbfsPerfTracker();
     this.abfsCounters = abfsClientContext.getAbfsCounters();
+
+    ThreadFactory tf =
+        new ThreadFactoryBuilder().setNameFormat("AbfsClient Lease Ops").setDaemon(true).build();
+    this.executorService = MoreExecutors.listeningDecorator(
+        HadoopExecutors.newScheduledThreadPool(this.abfsConfiguration.getNumLeaseThreads(), tf));
   }
 
   public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials,
@@ -129,6 +149,7 @@ public void close() throws IOException {
     if (tokenProvider instanceof Closeable) {
       IOUtils.cleanupWithLogger(LOG, (Closeable) tokenProvider);
     }
+    HadoopExecutors.shutdown(executorService, LOG, 0, TimeUnit.SECONDS);
   }
 
   public String getFileSystem() {
@@ -317,6 +338,83 @@ public AbfsRestOperation createPath(final String path, final boolean isFile, fin
     return op;
   }
 
+  public AbfsRestOperation acquireLease(final String path, int duration) throws AzureBlobFileSystemException {
+    final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
+
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ACTION, ACQUIRE_LEASE_ACTION));
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_DURATION, Integer.toString(duration)));
+    requestHeaders.add(new AbfsHttpHeader(X_MS_PROPOSED_LEASE_ID, UUID.randomUUID().toString()));
+
+    final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
+
+    final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString());
+    final AbfsRestOperation op = new AbfsRestOperation(
+        AbfsRestOperationType.LeasePath,
+        this,
+        HTTP_METHOD_POST,
+        url,
+        requestHeaders);
+    op.execute();
+    return op;
+  }
+
+  public AbfsRestOperation renewLease(final String path, final String leaseId) throws AzureBlobFileSystemException {
+    final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
+
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ACTION, RENEW_LEASE_ACTION));
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ID, leaseId));
+
+    final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
+
+    final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString());
+    final AbfsRestOperation op = new AbfsRestOperation(
+        AbfsRestOperationType.LeasePath,
+        this,
+        HTTP_METHOD_POST,
+        url,
+        requestHeaders);
+    op.execute();
+    return op;
+  }
+
+  public AbfsRestOperation releaseLease(final String path, final String leaseId) throws AzureBlobFileSystemException {
+    final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
+
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ACTION, RELEASE_LEASE_ACTION));
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ID, leaseId));
+
+    final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
+
+    final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString());
+    final AbfsRestOperation op = new AbfsRestOperation(
+        AbfsRestOperationType.LeasePath,
+        this,
+        HTTP_METHOD_POST,
+        url,
+        requestHeaders);
+    op.execute();
+    return op;
+  }
+
+  public AbfsRestOperation breakLease(final String path) throws AzureBlobFileSystemException {
+    final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
+
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ACTION, BREAK_LEASE_ACTION));
+    requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_BREAK_PERIOD, DEFAULT_LEASE_BREAK_PERIOD));
+
+    final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
+
+    final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString());
+    final AbfsRestOperation op = new AbfsRestOperation(
+        AbfsRestOperationType.LeasePath,
+        this,
+        HTTP_METHOD_POST,
+        url,
+        requestHeaders);
+    op.execute();
+    return op;
+  }
+
   public AbfsRestOperation renamePath(String source, final String destination, final String continuation)
           throws AzureBlobFileSystemException {
     final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
@@ -416,6 +514,9 @@ public AbfsRestOperation append(final String path, final byte[] buffer,
     // PUT and specify the real method in the X-Http-Method-Override header.
     requestHeaders.add(new AbfsHttpHeader(X_HTTP_METHOD_OVERRIDE,
         HTTP_METHOD_PATCH));
+    if (reqParams.getLeaseId() != null) {
+      requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ID, reqParams.getLeaseId()));
+    }
 
     final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
     abfsUriQueryBuilder.addQuery(QUERY_PARAM_ACTION, APPEND_ACTION);
@@ -492,13 +593,16 @@ public boolean appendSuccessCheckOp(AbfsRestOperation op, final String path,
   }
 
   public AbfsRestOperation flush(final String path, final long position, boolean retainUncommittedData,
-                                 boolean isClose, final String cachedSasToken)
+                                 boolean isClose, final String cachedSasToken, final String leaseId)
       throws AzureBlobFileSystemException {
     final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
     // JDK7 does not support PATCH, so to workaround the issue we will use
     // PUT and specify the real method in the X-Http-Method-Override header.
     requestHeaders.add(new AbfsHttpHeader(X_HTTP_METHOD_OVERRIDE,
             HTTP_METHOD_PATCH));
+    if (leaseId != null) {
+      requestHeaders.add(new AbfsHttpHeader(X_MS_LEASE_ID, leaseId));
+    }
 
     final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder();
     abfsUriQueryBuilder.addQuery(QUERY_PARAM_ACTION, FLUSH_ACTION);
@@ -1003,4 +1107,21 @@ public SASTokenProvider getSasTokenProvider() {
   protected AbfsCounters getAbfsCounters() {
     return abfsCounters;
   }
+
+  public int getNumLeaseThreads() {
+    return abfsConfiguration.getNumLeaseThreads();
+  }
+
+  public <V> ListenableScheduledFuture<V> schedule(Callable<V> callable, long delay,
+      TimeUnit timeUnit) {
+    return executorService.schedule(callable, delay, timeUnit);
+  }
+
+  public ListenableFuture<?> submit(Runnable runnable) {
+    return executorService.submit(runnable);
+  }
+
+  public <V> void addCallback(ListenableFuture<V> future, FutureCallback<V> callback) {
+    Futures.addCallback(future, callback, executorService);
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java
new file mode 100644
index 0000000000000..e15795efee68d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_LEASE_THREADS;
+
+/**
+ * ABFS error constants.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class AbfsErrors {
+  public static final String ERR_WRITE_WITHOUT_LEASE = "Attempted to write to file without lease";
+  public static final String ERR_LEASE_EXPIRED = "A lease ID was specified, but the lease for the"
+      + " resource has expired";
+  public static final String ERR_NO_LEASE_ID_SPECIFIED = "There is currently a lease on the "
+      + "resource and no lease ID was specified in the request";
+  public static final String ERR_PARALLEL_ACCESS_DETECTED = "Parallel access to the create path "
+      + "detected. Failing request to honor single writer semantics";
+  public static final String ERR_ACQUIRING_LEASE = "Unable to acquire lease";
+  public static final String ERR_LEASE_ALREADY_PRESENT = "There is already a lease present";
+  public static final String ERR_LEASE_NOT_PRESENT = "There is currently no lease on the resource";
+  public static final String ERR_LEASE_ID_NOT_PRESENT = "The lease ID is not present with the "
+      + "specified lease operation";
+  public static final String ERR_LEASE_DID_NOT_MATCH = "The lease ID specified did not match the "
+      + "lease ID for the resource with the specified lease operation";
+  public static final String ERR_LEASE_BROKEN = "The lease ID matched, but the lease has been "
+    + "broken explicitly and cannot be renewed";
+  public static final String ERR_LEASE_FUTURE_EXISTS = "There is already an existing lease "
+      + "operation";
+  public static final String ERR_NO_LEASE_THREADS = "Lease desired but no lease threads "
+      + "configured, set " + FS_AZURE_LEASE_THREADS;
+
+  private AbfsErrors() {}
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java
new file mode 100644
index 0000000000000..97a8b0228a5b3
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableScheduledFuture;
+import org.apache.hadoop.thirdparty.org.checkerframework.checker.nullness.qual.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.INFINITE_LEASE_DURATION;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_ACQUIRING_LEASE;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_LEASE_FUTURE_EXISTS;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_NO_LEASE_THREADS;
+
+/**
+ * AbfsLease manages an Azure blob lease. It acquires an infinite lease on instantiation and
+ * releases the lease when free() is called. Use it to prevent writes to the blob by other
+ * processes that don't have the lease.
+ *
+ * Creating a new Lease object blocks the caller until the Azure blob lease is acquired. It will
+ * retry a fixed number of times before failing if there is a problem acquiring the lease.
+ *
+ * Call free() to release the Lease. If the holder process dies, AzureBlobFileSystem breakLease
+ * will need to be called before another client will be able to write to the file.
+ */
+public final class AbfsLease {
+  private static final Logger LOG = LoggerFactory.getLogger(AbfsLease.class);
+
+  // Number of retries for acquiring lease
+  static final int DEFAULT_LEASE_ACQUIRE_MAX_RETRIES = 7;
+  // Retry interval for acquiring lease in secs
+  static final int DEFAULT_LEASE_ACQUIRE_RETRY_INTERVAL = 10;
+
+  private final AbfsClient client;
+  private final String path;
+
+  // Lease status variables
+  private volatile boolean leaseFreed;
+  private volatile String leaseID = null;
+  private volatile Throwable exception = null;
+  private volatile int acquireRetryCount = 0;
+  private volatile ListenableScheduledFuture<AbfsRestOperation> future = null;
+
+  public static class LeaseException extends AzureBlobFileSystemException {
+    public LeaseException(Throwable t) {
+      super(ERR_ACQUIRING_LEASE + ": " + t, t);
+    }
+
+    public LeaseException(String s) {
+      super(s);
+    }
+  }
+
+  public AbfsLease(AbfsClient client, String path) throws AzureBlobFileSystemException {
+    this(client, path, DEFAULT_LEASE_ACQUIRE_MAX_RETRIES, DEFAULT_LEASE_ACQUIRE_RETRY_INTERVAL);
+  }
+
+  @VisibleForTesting
+  public AbfsLease(AbfsClient client, String path, int acquireMaxRetries,
+      int acquireRetryInterval) throws AzureBlobFileSystemException {
+    this.leaseFreed = false;
+    this.client = client;
+    this.path = path;
+
+    if (client.getNumLeaseThreads() < 1) {
+      throw new LeaseException(ERR_NO_LEASE_THREADS);
+    }
+
+    // Try to get the lease a specified number of times, else throw an error
+    RetryPolicy retryPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
+        acquireMaxRetries, acquireRetryInterval, TimeUnit.SECONDS);
+    acquireLease(retryPolicy, 0, acquireRetryInterval, 0);
+
+    while (leaseID == null && exception == null) {
+      try {
+        future.get();
+      } catch (Exception e) {
+        LOG.debug("Got exception waiting for acquire lease future. Checking if lease ID or "
+            + "exception have been set", e);
+      }
+    }
+    if (exception != null) {
+      LOG.error("Failed to acquire lease on {}", path);
+      throw new LeaseException(exception);
+    }
+
+    LOG.debug("Acquired lease {} on {}", leaseID, path);
+  }
+
+  private void acquireLease(RetryPolicy retryPolicy, int numRetries, int retryInterval, long delay)
+      throws LeaseException {
+    LOG.debug("Attempting to acquire lease on {}, retry {}", path, numRetries);
+    if (future != null && !future.isDone()) {
+      throw new LeaseException(ERR_LEASE_FUTURE_EXISTS);
+    }
+    future = client.schedule(() -> client.acquireLease(path, INFINITE_LEASE_DURATION),
+        delay, TimeUnit.SECONDS);
+    client.addCallback(future, new FutureCallback<AbfsRestOperation>() {
+      @Override
+      public void onSuccess(@Nullable AbfsRestOperation op) {
+        leaseID = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_LEASE_ID);
+        LOG.debug("Acquired lease {} on {}", leaseID, path);
+      }
+
+      @Override
+      public void onFailure(Throwable throwable) {
+        try {
+          if (RetryPolicy.RetryAction.RetryDecision.RETRY
+              == retryPolicy.shouldRetry(null, numRetries, 0, true).action) {
+            LOG.debug("Failed to acquire lease on {}, retrying: {}", path, throwable);
+            acquireRetryCount++;
+            acquireLease(retryPolicy, numRetries + 1, retryInterval, retryInterval);
+          } else {
+            exception = throwable;
+          }
+        } catch (Exception e) {
+          exception = throwable;
+        }
+      }
+    });
+  }
+
+  /**
+   * Cancel future and free the lease. If an exception occurs while releasing the lease, the error
+   * will be logged. If the lease cannot be released, AzureBlobFileSystem breakLease will need to
+   * be called before another client will be able to write to the file.
+   */
+  public void free() {
+    if (leaseFreed) {
+      return;
+    }
+    try {
+      LOG.debug("Freeing lease: path {}, lease id {}", path, leaseID);
+      if (future != null && !future.isDone()) {
+        future.cancel(true);
+      }
+      client.releaseLease(path, leaseID);
+    } catch (IOException e) {
+      LOG.warn("Exception when trying to release lease {} on {}. Lease will need to be broken: {}",
+          leaseID, path, e.getMessage());
+    } finally {
+      // Even if releasing the lease fails (e.g. because the file was deleted),
+      // make sure to record that we freed the lease
+      leaseFreed = true;
+      LOG.debug("Freed lease {} on {}", leaseID, path);
+    }
+  }
+
+  public boolean isFreed() {
+    return leaseFreed;
+  }
+
+  public String getLeaseID() {
+    return leaseID;
+  }
+
+  @VisibleForTesting
+  public int getAcquireRetryCount() {
+    return acquireRetryCount;
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
index 2d02019ab11c6..80b35ee4d3a91 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
@@ -37,6 +37,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
 import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
@@ -53,6 +54,7 @@
 import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.Syncable;
 
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_WRITE_WITHOUT_LEASE;
 import static org.apache.hadoop.fs.impl.StoreImplementationUtils.isProbeForSyncable;
 import static org.apache.hadoop.io.IOUtils.wrapException;
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters.Mode.APPEND_MODE;
@@ -92,6 +94,9 @@ public class AbfsOutputStream extends OutputStream implements Syncable,
   // SAS tokens can be re-used until they expire
   private CachedSASToken cachedSasToken;
 
+  private AbfsLease lease;
+  private String leaseId;
+
   /**
    * Queue storing buffers with the size of the Azure block ready for
    * reuse. The pool allows reusing the blocks instead of allocating new
@@ -142,6 +147,10 @@ public AbfsOutputStream(
     }
     this.maxRequestsThatCanBeQueued = abfsOutputStreamContext
         .getMaxWriteRequestsToQueue();
+
+    this.lease = abfsOutputStreamContext.getLease();
+    this.leaseId = abfsOutputStreamContext.getLeaseId();
+
     this.threadExecutor
         = new ThreadPoolExecutor(maxConcurrentRequestCount,
         maxConcurrentRequestCount,
@@ -203,6 +212,10 @@ public synchronized void write(final byte[] data, final int off, final int lengt
       throw new IndexOutOfBoundsException();
     }
 
+    if (hasLease() && isLeaseFreed()) {
+      throw new PathIOException(path, ERR_WRITE_WITHOUT_LEASE);
+    }
+
     int currentOffset = off;
     int writableBytes = bufferSize - bufferIndex;
     int numberOfBytesToWrite = length;
@@ -306,6 +319,10 @@ public synchronized void close() throws IOException {
       // See HADOOP-16785
       throw wrapException(path, e.getMessage(), e);
     } finally {
+      if (hasLease()) {
+        lease.free();
+        lease = null;
+      }
       lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
       buffer = null;
       bufferIndex = 0;
@@ -372,7 +389,7 @@ private void writeAppendBlobCurrentBufferToService() throws IOException {
     try (AbfsPerfInfo perfInfo = new AbfsPerfInfo(tracker,
             "writeCurrentBufferToService", "append")) {
       AppendRequestParameters reqParams = new AppendRequestParameters(offset, 0,
-          bytesLength, APPEND_MODE, true);
+          bytesLength, APPEND_MODE, true, leaseId);
       AbfsRestOperation op = client.append(path, bytes, reqParams, cachedSasToken.get());
       cachedSasToken.update(op.getSasToken());
       if (outputStreamStatistics != null) {
@@ -448,7 +465,7 @@ private synchronized void writeCurrentBufferToService(boolean isFlush, boolean i
                       mode = FLUSH_MODE;
                     }
                     AppendRequestParameters reqParams = new AppendRequestParameters(
-                        offset, 0, bytesLength, mode, false);
+                        offset, 0, bytesLength, mode, false, leaseId);
                     AbfsRestOperation op = client.append(path, bytes, reqParams,
                         cachedSasToken.get());
                     cachedSasToken.update(op.getSasToken());
@@ -517,7 +534,8 @@ private synchronized void flushWrittenBytesToServiceInternal(final long offset,
     AbfsPerfTracker tracker = client.getAbfsPerfTracker();
     try (AbfsPerfInfo perfInfo = new AbfsPerfInfo(tracker,
             "flushWrittenBytesToServiceInternal", "flush")) {
-      AbfsRestOperation op = client.flush(path, offset, retainUncommitedData, isClose, cachedSasToken.get());
+      AbfsRestOperation op = client.flush(path, offset, retainUncommitedData, isClose,
+          cachedSasToken.get(), leaseId);
       cachedSasToken.update(op.getSasToken());
       perfInfo.registerResult(op.getResult()).registerSuccess(true);
     } catch (AzureBlobFileSystemException ex) {
@@ -637,6 +655,19 @@ public IOStatistics getIOStatistics() {
     return ioStatistics;
   }
 
+  @VisibleForTesting
+  public boolean isLeaseFreed() {
+    if (lease == null) {
+      return true;
+    }
+    return lease.isFreed();
+  }
+
+  @VisibleForTesting
+  public boolean hasLease() {
+    return lease != null;
+  }
+
   /**
    * Appending AbfsOutputStream statistics to base toString().
    *
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStreamContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStreamContext.java
index 925cd4f7b5646..48f6f54081053 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStreamContext.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStreamContext.java
@@ -39,6 +39,8 @@ public class AbfsOutputStreamContext extends AbfsStreamContext {
 
   private int maxWriteRequestsToQueue;
 
+  private AbfsLease lease;
+
   public AbfsOutputStreamContext(final long sasTokenRenewPeriodForStreamsInSeconds) {
     super(sasTokenRenewPeriodForStreamsInSeconds);
   }
@@ -94,6 +96,11 @@ public AbfsOutputStreamContext withMaxWriteRequestsToQueue(
     return this;
   }
 
+  public AbfsOutputStreamContext withLease(final AbfsLease lease) {
+    this.lease = lease;
+    return this;
+  }
+
   public int getWriteBufferSize() {
     return writeBufferSize;
   }
@@ -125,4 +132,15 @@ public int getMaxWriteRequestsToQueue() {
   public boolean isEnableSmallWriteOptimization() {
     return this.enableSmallWriteOptimization;
   }
+
+  public AbfsLease getLease() {
+    return this.lease;
+  }
+
+  public String getLeaseId() {
+    if (this.lease == null) {
+      return null;
+    }
+    return this.lease.getLeaseID();
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index 584b71f1ee5bf..b046cbc03a30b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -131,6 +131,7 @@ String getSasToken() {
     this.url = url;
     this.requestHeaders = requestHeaders;
     this.hasRequestBody = (AbfsHttpConstants.HTTP_METHOD_PUT.equals(method)
+            || AbfsHttpConstants.HTTP_METHOD_POST.equals(method)
             || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method));
     this.sasToken = sasToken;
     this.abfsCounters = client.getAbfsCounters();
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationType.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationType.java
index d3031860dd1c2..830297f381b91 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationType.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationType.java
@@ -40,5 +40,6 @@ public enum AbfsRestOperationType {
     Flush,
     ReadFile,
     DeletePath,
-    CheckAccess
+    CheckAccess,
+    LeasePath,
 }
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
index 33d4a0fa428a0..6be5952b03aa6 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
@@ -887,6 +887,22 @@ enabled for your Azure Storage account."
 The directories can be specified as comma separated values. By default the value
 is "/hbase"
 
+### <a name="infiniteleaseoptions"></a> Infinite Lease Options
+`fs.azure.infinite-lease.directories`: Directories for infinite lease support
+can be specified comma separated in this config. By default, multiple
+clients will be able to write to the same file simultaneously. When writing
+to files contained within the directories specified in this config, the
+client will obtain a lease on the file that will prevent any other clients
+from writing to the file. When the output stream is closed, the lease will be
+released. To revoke a client's write access for a file, the
+AzureBlobFilesystem breakLease method may be called. If the client dies
+before the file can be closed and the lease released, breakLease will need to
+be called before another client will be able to write to the file.
+
+`fs.azure.lease.threads`: This is the size of the thread pool that will be
+used for lease operations for infinite lease directories. By default the value
+is 0, so it must be set to at least 1 to support infinite lease directories.
+
 ### <a name="perfoptions"></a> Perf Options
 
 #### <a name="abfstracklatencyoptions"></a> 1. HTTP Request Tracking Options
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java
new file mode 100644
index 0000000000000..9857da8957e22
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.util.concurrent.RejectedExecutionException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsLease;
+import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_INFINITE_LEASE_KEY;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_LEASE_THREADS;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_ACQUIRING_LEASE;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_LEASE_EXPIRED;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_LEASE_NOT_PRESENT;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_NO_LEASE_ID_SPECIFIED;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_NO_LEASE_THREADS;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_PARALLEL_ACCESS_DETECTED;
+
+/**
+ * Test lease operations.
+ */
+public class ITestAzureBlobFileSystemLease extends AbstractAbfsIntegrationTest {
+  private static final int TEST_EXECUTION_TIMEOUT = 30 * 1000;
+  private static final int LONG_TEST_EXECUTION_TIMEOUT = 90 * 1000;
+  private static final String TEST_FILE = "testfile";
+  private final boolean isHNSEnabled;
+
+  public ITestAzureBlobFileSystemLease() throws Exception {
+    super();
+
+    this.isHNSEnabled = getConfiguration()
+        .getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
+  }
+
+  private AzureBlobFileSystem getCustomFileSystem(Path infiniteLeaseDirs, int numLeaseThreads) throws Exception {
+    Configuration conf = getRawConfiguration();
+    conf.setBoolean(String.format("fs.%s.impl.disable.cache", getAbfsScheme()), true);
+    conf.set(FS_AZURE_INFINITE_LEASE_KEY, infiniteLeaseDirs.toUri().getPath());
+    conf.setInt(FS_AZURE_LEASE_THREADS, numLeaseThreads);
+    return getFileSystem(conf);
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testNoInfiniteLease() throws IOException {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getFileSystem();
+    fs.mkdirs(testFilePath.getParent());
+    try (FSDataOutputStream out = fs.create(testFilePath)) {
+      Assert.assertFalse("Output stream should not have lease",
+          ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+    }
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testNoLeaseThreads() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 0);
+    fs.mkdirs(testFilePath.getParent());
+    LambdaTestUtils.intercept(IOException.class, ERR_NO_LEASE_THREADS, () -> {
+      try (FSDataOutputStream out = fs.create(testFilePath)) {
+      }
+      return "No failure when lease requested with 0 lease threads";
+    });
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testOneWriter() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    FSDataOutputStream out = fs.create(testFilePath);
+    Assert.assertTrue("Output stream should have lease",
+        ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+    out.close();
+    Assert.assertFalse("Output stream should not have lease",
+        ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testSubDir() throws Exception {
+    final Path testFilePath = new Path(new Path(path(methodName.getMethodName()), "subdir"),
+        TEST_FILE);
+    final AzureBlobFileSystem fs =
+        getCustomFileSystem(testFilePath.getParent().getParent(), 1);
+    fs.mkdirs(testFilePath.getParent().getParent());
+
+    FSDataOutputStream out = fs.create(testFilePath);
+    Assert.assertTrue("Output stream should have lease",
+        ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+    out.close();
+    Assert.assertFalse("Output stream should not have lease",
+        ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testTwoCreate() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    try (FSDataOutputStream out = fs.create(testFilePath)) {
+      LambdaTestUtils.intercept(IOException.class, isHNSEnabled ? ERR_PARALLEL_ACCESS_DETECTED
+          : ERR_NO_LEASE_ID_SPECIFIED, () -> {
+        try (FSDataOutputStream out2 = fs.create(testFilePath)) {
+        }
+        return "Expected second create on infinite lease dir to fail";
+      });
+    }
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  private void twoWriters(AzureBlobFileSystem fs, Path testFilePath, boolean expectException) throws Exception {
+    try (FSDataOutputStream out = fs.create(testFilePath)) {
+      try (FSDataOutputStream out2 = fs.append(testFilePath)) {
+        out2.writeInt(2);
+        out2.hsync();
+      } catch (IOException e) {
+        if (expectException) {
+          GenericTestUtils.assertExceptionContains(ERR_ACQUIRING_LEASE, e);
+        } else {
+          throw e;
+        }
+      }
+      out.writeInt(1);
+      out.hsync();
+    }
+
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testTwoWritersCreateAppendNoInfiniteLease() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getFileSystem();
+    fs.mkdirs(testFilePath.getParent());
+
+    twoWriters(fs, testFilePath, false);
+  }
+
+  @Test(timeout = LONG_TEST_EXECUTION_TIMEOUT)
+  public void testTwoWritersCreateAppendWithInfiniteLeaseEnabled() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    twoWriters(fs, testFilePath, true);
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testLeaseFreedOnClose() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    FSDataOutputStream out;
+    out = fs.create(testFilePath);
+    out.write(0);
+    Assert.assertTrue("Output stream should have lease",
+        ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+    out.close();
+    Assert.assertFalse("Output stream should not have lease after close",
+        ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testWriteAfterBreakLease() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    FSDataOutputStream out;
+    out = fs.create(testFilePath);
+    out.write(0);
+    out.hsync();
+
+    fs.breakLease(testFilePath);
+
+    LambdaTestUtils.intercept(IOException.class, ERR_LEASE_EXPIRED, () -> {
+      out.write(1);
+      out.hsync();
+      return "Expected exception on write after lease break but got " + out;
+    });
+
+    LambdaTestUtils.intercept(IOException.class, ERR_LEASE_EXPIRED, () -> {
+      out.close();
+      return "Expected exception on close after lease break but got " + out;
+    });
+
+    Assert.assertTrue("Output stream lease should be freed",
+        ((AbfsOutputStream) out.getWrappedStream()).isLeaseFreed());
+
+    try (FSDataOutputStream out2 = fs.append(testFilePath)) {
+      out2.write(2);
+      out2.hsync();
+    }
+
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = LONG_TEST_EXECUTION_TIMEOUT)
+  public void testLeaseFreedAfterBreak() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    FSDataOutputStream out = fs.create(testFilePath);
+    out.write(0);
+
+    fs.breakLease(testFilePath);
+
+    LambdaTestUtils.intercept(IOException.class, ERR_LEASE_EXPIRED, () -> {
+      out.close();
+      return "Expected exception on close after lease break but got " + out;
+    });
+
+    Assert.assertTrue("Output stream lease should be freed",
+        ((AbfsOutputStream) out.getWrappedStream()).isLeaseFreed());
+
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testInfiniteLease() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    try (FSDataOutputStream out = fs.create(testFilePath)) {
+      Assert.assertTrue("Output stream should have lease",
+          ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+      out.write(0);
+    }
+    Assert.assertTrue(fs.getAbfsStore().areLeasesFreed());
+
+    try (FSDataOutputStream out = fs.append(testFilePath)) {
+      Assert.assertTrue("Output stream should have lease",
+          ((AbfsOutputStream) out.getWrappedStream()).hasLease());
+      out.write(1);
+    }
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testFileSystemClose() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+
+    FSDataOutputStream out = fs.create(testFilePath);
+    out.write(0);
+    Assert.assertFalse("Store leases should exist", fs.getAbfsStore().areLeasesFreed());
+    fs.close();
+    Assert.assertTrue("Store leases were not freed", fs.getAbfsStore().areLeasesFreed());
+
+    LambdaTestUtils.intercept(IOException.class, isHNSEnabled ? ERR_LEASE_NOT_PRESENT
+        : ERR_LEASE_EXPIRED, () -> {
+      out.close();
+      return "Expected exception on close after closed FS but got " + out;
+    });
+
+    LambdaTestUtils.intercept(RejectedExecutionException.class, () -> {
+      try (FSDataOutputStream out2 = fs.append(testFilePath)) {
+      }
+      return "Expected exception on new append after closed FS";
+    });
+  }
+
+  @Test(timeout = TEST_EXECUTION_TIMEOUT)
+  public void testAcquireRetry() throws Exception {
+    final Path testFilePath = new Path(path(methodName.getMethodName()), TEST_FILE);
+    final AzureBlobFileSystem fs = getCustomFileSystem(testFilePath.getParent(), 1);
+    fs.mkdirs(testFilePath.getParent());
+    fs.createNewFile(testFilePath);
+
+    AbfsLease lease = new AbfsLease(fs.getAbfsClient(), testFilePath.toUri().getPath());
+    Assert.assertNotNull("Did not successfully lease file", lease.getLeaseID());
+    lease.free();
+    Assert.assertEquals("Unexpected acquire retry count", 0, lease.getAcquireRetryCount());
+
+    AbfsClient mockClient = spy(fs.getAbfsClient());
+
+    doThrow(new AbfsLease.LeaseException("failed to acquire 1"))
+        .doThrow(new AbfsLease.LeaseException("failed to acquire 2"))
+        .doCallRealMethod()
+        .when(mockClient).acquireLease(anyString(), anyInt());
+
+    lease = new AbfsLease(mockClient, testFilePath.toUri().getPath(), 5, 1);
+    Assert.assertNotNull("Acquire lease should have retried", lease.getLeaseID());
+    lease.free();
+    Assert.assertEquals("Unexpected acquire retry count", 2, lease.getAcquireRetryCount());
+
+    doThrow(new AbfsLease.LeaseException("failed to acquire"))
+        .when(mockClient).acquireLease(anyString(), anyInt());
+
+    LambdaTestUtils.intercept(AzureBlobFileSystemException.class, () -> {
+      new AbfsLease(mockClient, testFilePath.toUri().getPath(), 5, 1);
+    });
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
index f02eadc9a0491..6a02435fc6e5e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
@@ -24,11 +24,14 @@
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
 import org.apache.hadoop.fs.azurebfs.utils.Base64;
 
-import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
-import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_LEASE_DURATION;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE;
-
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.INFINITE_LEASE_DURATION;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_LEASE_DURATION;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_LEASE_DURATION;
 
 /**
  * Test configuration validators.
@@ -58,6 +61,26 @@ public void testIntegerConfigValidatorThrowsIfMissingValidValue() throws Excepti
     integerConfigurationValidator.validate("3072");
   }
 
+  @Test
+  public void testIntegerWithOutlierConfigValidator() throws Exception {
+    IntegerConfigurationBasicValidator integerConfigurationValidator = new IntegerConfigurationBasicValidator(
+        INFINITE_LEASE_DURATION, MIN_LEASE_DURATION, MAX_LEASE_DURATION, DEFAULT_LEASE_DURATION, FAKE_KEY,
+        false);
+
+    assertEquals(INFINITE_LEASE_DURATION, (int) integerConfigurationValidator.validate("-1"));
+    assertEquals(DEFAULT_LEASE_DURATION, (int) integerConfigurationValidator.validate(null));
+    assertEquals(MIN_LEASE_DURATION, (int) integerConfigurationValidator.validate("15"));
+    assertEquals(MAX_LEASE_DURATION, (int) integerConfigurationValidator.validate("60"));
+  }
+
+  @Test(expected = InvalidConfigurationValueException.class)
+  public void testIntegerWithOutlierConfigValidatorThrowsIfMissingValidValue() throws Exception {
+    IntegerConfigurationBasicValidator integerConfigurationValidator = new IntegerConfigurationBasicValidator(
+        INFINITE_LEASE_DURATION, MIN_LEASE_DURATION, MAX_LEASE_DURATION, DEFAULT_LEASE_DURATION, FAKE_KEY,
+        true);
+    integerConfigurationValidator.validate("14");
+  }
+
   @Test
   public void testLongConfigValidator() throws Exception {
     LongConfigurationBasicValidator longConfigurationValidator = new LongConfigurationBasicValidator(
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsOutputStream.java
index 1e6b8efe6d9d2..f4243bc7e287b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsOutputStream.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.ArgumentMatchers.refEq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -86,7 +87,7 @@ public void verifyShortWriteRequest() throws Exception {
     AbfsPerfTracker tracker = new AbfsPerfTracker("test", accountName1, abfsConf);
     when(client.getAbfsPerfTracker()).thenReturn(tracker);
     when(client.append(anyString(), any(byte[].class), any(AppendRequestParameters.class), any())).thenReturn(op);
-    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any())).thenReturn(op);
+    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any(), isNull())).thenReturn(op);
 
     AbfsOutputStream out = new AbfsOutputStream(client, null, PATH, 0,
         populateAbfsOutputStreamContext(BUFFER_SIZE, true, false, false));
@@ -104,9 +105,9 @@ public void verifyShortWriteRequest() throws Exception {
     out.hsync();
 
     AppendRequestParameters firstReqParameters = new AppendRequestParameters(
-        0, 0, WRITE_SIZE, APPEND_MODE, false);
+        0, 0, WRITE_SIZE, APPEND_MODE, false, null);
     AppendRequestParameters secondReqParameters = new AppendRequestParameters(
-        WRITE_SIZE, 0, 2 * WRITE_SIZE, APPEND_MODE, false);
+        WRITE_SIZE, 0, 2 * WRITE_SIZE, APPEND_MODE, false, null);
 
     verify(client, times(1)).append(
         eq(PATH), any(byte[].class), refEq(firstReqParameters), any());
@@ -133,7 +134,7 @@ public void verifyWriteRequest() throws Exception {
 
     when(client.getAbfsPerfTracker()).thenReturn(tracker);
     when(client.append(anyString(), any(byte[].class), any(AppendRequestParameters.class), any())).thenReturn(op);
-    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any())).thenReturn(op);
+    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any(), isNull())).thenReturn(op);
 
     AbfsOutputStream out = new AbfsOutputStream(client, null, PATH, 0,
         populateAbfsOutputStreamContext(BUFFER_SIZE, true, false, false));
@@ -146,9 +147,9 @@ public void verifyWriteRequest() throws Exception {
     out.close();
 
     AppendRequestParameters firstReqParameters = new AppendRequestParameters(
-        0, 0, BUFFER_SIZE, APPEND_MODE, false);
+        0, 0, BUFFER_SIZE, APPEND_MODE, false, null);
     AppendRequestParameters secondReqParameters = new AppendRequestParameters(
-        BUFFER_SIZE, 0, 5*WRITE_SIZE-BUFFER_SIZE, APPEND_MODE, false);
+        BUFFER_SIZE, 0, 5*WRITE_SIZE-BUFFER_SIZE, APPEND_MODE, false, null);
 
     verify(client, times(1)).append(
         eq(PATH), any(byte[].class), refEq(firstReqParameters), any());
@@ -165,7 +166,7 @@ public void verifyWriteRequest() throws Exception {
     ArgumentCaptor<String> acFlushSASToken = ArgumentCaptor.forClass(String.class);
 
     verify(client, times(1)).flush(acFlushPath.capture(), acFlushPosition.capture(), acFlushRetainUnCommittedData.capture(), acFlushClose.capture(),
-        acFlushSASToken.capture());
+        acFlushSASToken.capture(), isNull());
     assertThat(Arrays.asList(PATH)).describedAs("path").isEqualTo(acFlushPath.getAllValues());
     assertThat(Arrays.asList(Long.valueOf(5*WRITE_SIZE))).describedAs("position").isEqualTo(acFlushPosition.getAllValues());
     assertThat(Arrays.asList(false)).describedAs("RetainUnCommittedData flag").isEqualTo(acFlushRetainUnCommittedData.getAllValues());
@@ -189,7 +190,7 @@ public void verifyWriteRequestOfBufferSizeAndClose() throws Exception {
 
     when(client.getAbfsPerfTracker()).thenReturn(tracker);
     when(client.append(anyString(), any(byte[].class), any(AppendRequestParameters.class), any())).thenReturn(op);
-    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any())).thenReturn(op);
+    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any(), isNull())).thenReturn(op);
     when(op.getSasToken()).thenReturn("testToken");
     when(op.getResult()).thenReturn(httpOp);
 
@@ -204,9 +205,9 @@ public void verifyWriteRequestOfBufferSizeAndClose() throws Exception {
     out.close();
 
     AppendRequestParameters firstReqParameters = new AppendRequestParameters(
-        0, 0, BUFFER_SIZE, APPEND_MODE, false);
+        0, 0, BUFFER_SIZE, APPEND_MODE, false, null);
     AppendRequestParameters secondReqParameters = new AppendRequestParameters(
-        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false);
+        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false, null);
 
     verify(client, times(1)).append(
         eq(PATH), any(byte[].class), refEq(firstReqParameters), any());
@@ -223,7 +224,7 @@ public void verifyWriteRequestOfBufferSizeAndClose() throws Exception {
     ArgumentCaptor<String> acFlushSASToken = ArgumentCaptor.forClass(String.class);
 
     verify(client, times(1)).flush(acFlushPath.capture(), acFlushPosition.capture(), acFlushRetainUnCommittedData.capture(), acFlushClose.capture(),
-        acFlushSASToken.capture());
+        acFlushSASToken.capture(), isNull());
     assertThat(Arrays.asList(PATH)).describedAs("path").isEqualTo(acFlushPath.getAllValues());
     assertThat(Arrays.asList(Long.valueOf(2*BUFFER_SIZE))).describedAs("position").isEqualTo(acFlushPosition.getAllValues());
     assertThat(Arrays.asList(false)).describedAs("RetainUnCommittedData flag").isEqualTo(acFlushRetainUnCommittedData.getAllValues());
@@ -247,7 +248,7 @@ public void verifyWriteRequestOfBufferSize() throws Exception {
 
     when(client.getAbfsPerfTracker()).thenReturn(tracker);
     when(client.append(anyString(), any(byte[].class), any(AppendRequestParameters.class), any())).thenReturn(op);
-    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any())).thenReturn(op);
+    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any(), isNull())).thenReturn(op);
     when(op.getSasToken()).thenReturn("testToken");
     when(op.getResult()).thenReturn(httpOp);
 
@@ -262,9 +263,9 @@ public void verifyWriteRequestOfBufferSize() throws Exception {
     Thread.sleep(1000);
 
     AppendRequestParameters firstReqParameters = new AppendRequestParameters(
-        0, 0, BUFFER_SIZE, APPEND_MODE, false);
+        0, 0, BUFFER_SIZE, APPEND_MODE, false, null);
     AppendRequestParameters secondReqParameters = new AppendRequestParameters(
-        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false);
+        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false, null);
 
     verify(client, times(1)).append(
         eq(PATH), any(byte[].class), refEq(firstReqParameters), any());
@@ -291,7 +292,7 @@ public void verifyWriteRequestOfBufferSizeWithAppendBlob() throws Exception {
 
     when(client.getAbfsPerfTracker()).thenReturn(tracker);
     when(client.append(anyString(), any(byte[].class), any(AppendRequestParameters.class), any())).thenReturn(op);
-    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any())).thenReturn(op);
+    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any(), isNull())).thenReturn(op);
 
     AbfsOutputStream out = new AbfsOutputStream(client, null, PATH, 0,
         populateAbfsOutputStreamContext(BUFFER_SIZE, true, false, true));
@@ -304,9 +305,9 @@ public void verifyWriteRequestOfBufferSizeWithAppendBlob() throws Exception {
     Thread.sleep(1000);
 
     AppendRequestParameters firstReqParameters = new AppendRequestParameters(
-        0, 0, BUFFER_SIZE, APPEND_MODE, true);
+        0, 0, BUFFER_SIZE, APPEND_MODE, true, null);
     AppendRequestParameters secondReqParameters = new AppendRequestParameters(
-        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, true);
+        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, true, null);
 
     verify(client, times(1)).append(
         eq(PATH), any(byte[].class), refEq(firstReqParameters), any());
@@ -334,7 +335,7 @@ public void verifyWriteRequestOfBufferSizeAndHFlush() throws Exception {
 
     when(client.getAbfsPerfTracker()).thenReturn(tracker);
     when(client.append(anyString(), any(byte[].class), any(AppendRequestParameters.class), any())).thenReturn(op);
-    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any())).thenReturn(op);
+    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any(), isNull())).thenReturn(op);
 
     AbfsOutputStream out = new AbfsOutputStream(client, null, PATH, 0,
         populateAbfsOutputStreamContext(BUFFER_SIZE, true, false, false));
@@ -347,9 +348,9 @@ public void verifyWriteRequestOfBufferSizeAndHFlush() throws Exception {
     out.hflush();
 
     AppendRequestParameters firstReqParameters = new AppendRequestParameters(
-        0, 0, BUFFER_SIZE, APPEND_MODE, false);
+        0, 0, BUFFER_SIZE, APPEND_MODE, false, null);
     AppendRequestParameters secondReqParameters = new AppendRequestParameters(
-        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false);
+        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false, null);
 
     verify(client, times(1)).append(
         eq(PATH), any(byte[].class), refEq(firstReqParameters), any());
@@ -366,7 +367,7 @@ public void verifyWriteRequestOfBufferSizeAndHFlush() throws Exception {
     ArgumentCaptor<String> acFlushSASToken = ArgumentCaptor.forClass(String.class);
 
     verify(client, times(1)).flush(acFlushPath.capture(), acFlushPosition.capture(), acFlushRetainUnCommittedData.capture(), acFlushClose.capture(),
-        acFlushSASToken.capture());
+        acFlushSASToken.capture(), isNull());
     assertThat(Arrays.asList(PATH)).describedAs("path").isEqualTo(acFlushPath.getAllValues());
     assertThat(Arrays.asList(Long.valueOf(2*BUFFER_SIZE))).describedAs("position").isEqualTo(acFlushPosition.getAllValues());
     assertThat(Arrays.asList(false)).describedAs("RetainUnCommittedData flag").isEqualTo(acFlushRetainUnCommittedData.getAllValues());
@@ -388,7 +389,7 @@ public void verifyWriteRequestOfBufferSizeAndFlush() throws Exception {
     AbfsPerfTracker tracker = new AbfsPerfTracker("test", accountName1, abfsConf);
     when(client.getAbfsPerfTracker()).thenReturn(tracker);
     when(client.append(anyString(), any(byte[].class), any(AppendRequestParameters.class), any())).thenReturn(op);
-    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any())).thenReturn(op);
+    when(client.flush(anyString(), anyLong(), anyBoolean(), anyBoolean(), any(), isNull())).thenReturn(op);
 
     AbfsOutputStream out = new AbfsOutputStream(client, null, PATH, 0,
         populateAbfsOutputStreamContext(BUFFER_SIZE, true, false, false));
@@ -403,9 +404,9 @@ public void verifyWriteRequestOfBufferSizeAndFlush() throws Exception {
     Thread.sleep(1000);
 
     AppendRequestParameters firstReqParameters = new AppendRequestParameters(
-        0, 0, BUFFER_SIZE, APPEND_MODE, false);
+        0, 0, BUFFER_SIZE, APPEND_MODE, false, null);
     AppendRequestParameters secondReqParameters = new AppendRequestParameters(
-        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false);
+        BUFFER_SIZE, 0, BUFFER_SIZE, APPEND_MODE, false, null);
 
     verify(client, times(1)).append(
         eq(PATH), any(byte[].class), refEq(firstReqParameters), any());

From 82462739f82a936a57223dd2bb4d8b71bb257ed9 Mon Sep 17 00:00:00 2001
From: Inigo Goiri <elgoiri@gmail.com>
Date: Mon, 12 Apr 2021 20:44:31 -0700
Subject: [PATCH 23/43] Revert "HDFS-15423 RBF: WebHDFS create shouldn't choose
 DN from all sub-clusters (#2605)" (#2900)

This reverts commit cb3ed32fe0d937980999d590deb35ad97d40f9e6.
---
 .../router/RouterWebHdfsMethods.java          |  47 +-----
 .../router/web/RouterWebHDFSContract.java     |   2 -
 .../federation/MiniRouterDFSCluster.java      |  19 ++-
 .../router/TestRouterWebHdfsMethods.java      | 147 ------------------
 4 files changed, 17 insertions(+), 198 deletions(-)
 delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
index afc4a3d8fac21..f6ac70c368a52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
@@ -93,7 +93,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -105,8 +104,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 /**
  * WebHDFS Router implementation. This is an extension of
@@ -456,33 +453,21 @@ private DatanodeInfo chooseDatanode(final Router router,
       final String path, final HttpOpParam.Op op, final long openOffset,
       final String excludeDatanodes) throws IOException {
     final RouterRpcServer rpcServer = getRPCServer(router);
-    DatanodeInfo[] dns = {};
-    String resolvedNs = "";
+    DatanodeInfo[] dns = null;
     try {
       dns = rpcServer.getCachedDatanodeReport(DatanodeReportType.LIVE);
     } catch (IOException e) {
       LOG.error("Cannot get the datanodes from the RPC server", e);
     }
 
-    if (op == PutOpParam.Op.CREATE) {
-      try {
-        resolvedNs = rpcServer.getCreateLocation(path).getNameserviceId();
-      } catch (IOException e) {
-        LOG.error("Cannot get the name service " +
-            "to create file for path {} ", path, e);
-      }
-    }
-
     HashSet<Node> excludes = new HashSet<Node>();
-    Collection<String> collection =
-        getTrimmedStringCollection(excludeDatanodes);
-    for (DatanodeInfo dn : dns) {
-      String ns = getNsFromDataNodeNetworkLocation(dn.getNetworkLocation());
-      if (collection.contains(dn.getName())) {
-        excludes.add(dn);
-      } else if (op == PutOpParam.Op.CREATE && !ns.equals(resolvedNs)) {
-        // for CREATE, the dest dn should be in the resolved ns
-        excludes.add(dn);
+    if (excludeDatanodes != null) {
+      Collection<String> collection =
+          getTrimmedStringCollection(excludeDatanodes);
+      for (DatanodeInfo dn : dns) {
+        if (collection.contains(dn.getName())) {
+          excludes.add(dn);
+        }
       }
     }
 
@@ -517,22 +502,6 @@ private DatanodeInfo chooseDatanode(final Router router,
     return getRandomDatanode(dns, excludes);
   }
 
-  /**
-   * Get the nameservice info from datanode network location.
-   * @param location network location with format `/ns0/rack1`
-   * @return nameservice this datanode is in
-   */
-  @VisibleForTesting
-  public static String getNsFromDataNodeNetworkLocation(String location) {
-    // network location should be in the format of /ns/rack
-    Pattern pattern = Pattern.compile("^/([^/]*)/");
-    Matcher matcher = pattern.matcher(location);
-    if (matcher.find()) {
-      return matcher.group(1);
-    }
-    return "";
-  }
-
   /**
    * Get a random Datanode from a subcluster.
    * @param dns Nodes to be chosen from.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
index 6b90faecc78f3..1d308073290d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
@@ -64,8 +64,6 @@ public static void createCluster(Configuration conf) throws IOException {
       conf.addResource(CONTRACT_WEBHDFS_XML);
 
       cluster = new MiniRouterDFSCluster(true, 2, conf);
-      cluster.setIndependentDNs();
-      cluster.setNumDatanodesPerNameservice(3);
 
       // Start NNs and DNs and wait until ready
       cluster.startCluster(conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
index 8a7a03e018b95..896d08f2c49b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
@@ -774,15 +774,6 @@ public void startCluster(Configuration overrideConf) {
       }
       topology.setFederation(true);
 
-      // Generate conf for namenodes and datanodes
-      String ns0 = nameservices.get(0);
-      Configuration nnConf = generateNamenodeConfiguration(ns0);
-      if (overrideConf != null) {
-        nnConf.addResource(overrideConf);
-        // Router also uses this configurations as initial values.
-        routerConf = new Configuration(overrideConf);
-      }
-
       // Set independent DNs across subclusters
       int numDNs = nameservices.size() * numDatanodesPerNameservice;
       Configuration[] dnConfs = null;
@@ -790,7 +781,7 @@ public void startCluster(Configuration overrideConf) {
         dnConfs = new Configuration[numDNs];
         int dnId = 0;
         for (String nsId : nameservices) {
-          Configuration subclusterConf = new Configuration(nnConf);
+          Configuration subclusterConf = new Configuration();
           subclusterConf.set(DFS_INTERNAL_NAMESERVICES_KEY, nsId);
           for (int i = 0; i < numDatanodesPerNameservice; i++) {
             dnConfs[dnId] = subclusterConf;
@@ -800,6 +791,14 @@ public void startCluster(Configuration overrideConf) {
       }
 
       // Start mini DFS cluster
+      String ns0 = nameservices.get(0);
+      Configuration nnConf = generateNamenodeConfiguration(ns0);
+      if (overrideConf != null) {
+        nnConf.addResource(overrideConf);
+        // Router also uses this configurations as initial values.
+        routerConf = new Configuration(overrideConf);
+      }
+
       cluster = new MiniDFSCluster.Builder(nnConf)
           .numDataNodes(numDNs)
           .nnTopology(topology)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
deleted file mode 100644
index 7028928041452..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.federation.router;
-
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createMountTableEntry;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.FileNotFoundException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Collections;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
-import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test suite for Router Web Hdfs methods.
- */
-public class TestRouterWebHdfsMethods {
-  static final Logger LOG =
-      LoggerFactory.getLogger(TestRouterWebHdfsMethods.class);
-
-  private static StateStoreDFSCluster cluster;
-  private static RouterContext router;
-  private static String httpUri;
-
-  @BeforeClass
-  public static void globalSetUp() throws Exception {
-    cluster = new StateStoreDFSCluster(false, 2);
-    Configuration conf = new RouterConfigBuilder()
-        .stateStore()
-        .rpc()
-        .http()
-        .admin()
-        .build();
-    cluster.addRouterOverrides(conf);
-    cluster.startCluster();
-    cluster.startRouters();
-    cluster.waitClusterUp();
-    router = cluster.getRandomRouter();
-    httpUri = "http://"+router.getHttpAddress();
-  }
-
-  @AfterClass
-  public static void tearDown() {
-    if (cluster != null) {
-      cluster.shutdown();
-      cluster = null;
-    }
-  }
-
-  @Test
-  public void testWebHdfsCreate() throws Exception {
-    // the file is created at default ns (ns0)
-    String path = "/tmp/file";
-    URL url = new URL(getUri(path));
-    LOG.info("URL: {}", url);
-    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestMethod("PUT");
-    assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
-    verifyFile("ns0", path, true);
-    verifyFile("ns1", path, false);
-    conn.disconnect();
-  }
-
-  @Test
-  public void testWebHdfsCreateWithMounts() throws Exception {
-    // the file is created at mounted ns (ns1)
-    String mountPoint = "/tmp-ns1";
-    String path = "/tmp-ns1/file";
-    createMountTableEntry(
-        router.getRouter(), mountPoint,
-        DestinationOrder.RANDOM, Collections.singletonList("ns1"));
-    URL url = new URL(getUri(path));
-    LOG.info("URL: {}", url);
-    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestMethod("PUT");
-    assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
-    verifyFile("ns1", path, true);
-    verifyFile("ns0", path, false);
-    conn.disconnect();
-  }
-
-  private String getUri(String path) {
-    final String user = System.getProperty("user.name");
-    final StringBuilder uri = new StringBuilder(httpUri);
-    uri.append("/webhdfs/v1").
-        append(path).
-        append("?op=CREATE").
-        append("&user.name=" + user);
-    return uri.toString();
-  }
-
-  private void verifyFile(String ns, String path, boolean shouldExist)
-      throws Exception {
-    FileSystem fs = cluster.getNamenode(ns, null).getFileSystem();
-    try {
-      fs.getFileStatus(new Path(path));
-      if (!shouldExist) {
-        fail(path + " should not exist in ns " + ns);
-      }
-    } catch (FileNotFoundException e) {
-      if (shouldExist) {
-        fail(path + " should exist in ns " + ns);
-      }
-    }
-  }
-
-  @Test
-  public void testGetNsFromDataNodeNetworkLocation() {
-    assertEquals("ns0", RouterWebHdfsMethods
-        .getNsFromDataNodeNetworkLocation("/ns0/rack-info1"));
-    assertEquals("ns0", RouterWebHdfsMethods
-        .getNsFromDataNodeNetworkLocation("/ns0/row1/rack-info1"));
-    assertEquals("", RouterWebHdfsMethods
-        .getNsFromDataNodeNetworkLocation("/row0"));
-    assertEquals("", RouterWebHdfsMethods
-        .getNsFromDataNodeNetworkLocation("whatever-rack-info1"));
-  }
-}

From 156ecc89be3ae1f42bde9c22ab5ba96cf60df3c6 Mon Sep 17 00:00:00 2001
From: Akira Ajisaka <aajisaka@apache.org>
Date: Tue, 13 Apr 2021 17:08:49 +0900
Subject: [PATCH 24/43] HADOOP-17630. [JDK 15] TestPrintableString fails due to
 Unicode 13.0 support. (#2890)

Reviewed-by: Wei-Chiu Chuang <weichiu@apache.org>
---
 .../java/org/apache/hadoop/fs/shell/TestPrintableString.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java
index 8e09fc29744fe..91bfdd6d3948c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java
@@ -76,8 +76,8 @@ public void testNonPrintableCharacters() throws Exception {
         "x\uDB80\uDC00y\uDBFF\uDFFDz\u1050", "x?y?z\u1050");
 
     // Unassigned Unicode
-    expect("Should replace unassigned U+30000 and U+DFFFF",
-        "-\uD880\uDC00-\uDB3F\uDFFF-", "-?-?-");
+    expect("Should replace unassigned U+DFFFF",
+        "-\uDB3F\uDFFF-", "-?-");
 
     // Standalone surrogate character (not in a pair)
     expect("Should replace standalone surrogate U+DB80", "x\uDB80yz", "x?yz");

From b088f46b684ccefb401b9a3be197d75e3bc64f6b Mon Sep 17 00:00:00 2001
From: Gautham B A <gautham.bangalore@gmail.com>
Date: Tue, 13 Apr 2021 21:24:45 +0530
Subject: [PATCH 25/43] HDFS-15971. Make mkstemp cross platform (#2898)

---
 .../src/CMakeLists.txt                        |  3 +-
 .../libhdfs-tests/test_libhdfs_mini_stress.c  |  4 +-
 .../libhdfspp/lib/x-platform/c-api/syscall.cc | 22 ++++++-
 .../libhdfspp/lib/x-platform/c-api/syscall.h  |  3 +
 .../native/libhdfspp/lib/x-platform/syscall.h | 29 +++++++++
 .../libhdfspp/lib/x-platform/syscall_linux.cc | 11 ++++
 .../lib/x-platform/syscall_windows.cc         | 30 +++++++++
 .../native/libhdfspp/tests/CMakeLists.txt     | 13 ++--
 .../libhdfspp/tests/configuration_test.cc     | 23 ++++---
 .../libhdfspp/tests/configuration_test.h      | 64 +++++++++++++++----
 .../libhdfspp/tests/hdfs_builder_test.cc      |  4 +-
 .../tests/hdfs_configuration_test.cc          | 16 ++---
 .../tests/x-platform/syscall_common_test.cc   | 18 ++++++
 13 files changed, 199 insertions(+), 41 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index baf5f8cc30e79..4e943de1773f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -89,7 +89,8 @@ function(build_libhdfs_test NAME LIBRARY)
             list(APPEND FILES ${CMAKE_SOURCE_DIR}/main/native/libhdfs-tests/${FIL})
         endif()
     endforeach()
-    add_executable("${NAME}_${LIBRARY}" ${FILES})
+    add_executable("${NAME}_${LIBRARY}" $<TARGET_OBJECTS:x_platform_obj_c_api> $<TARGET_OBJECTS:x_platform_obj> ${FILES})
+    target_include_directories("${NAME}_${LIBRARY}" PRIVATE main/native/libhdfspp/lib)
 endfunction()
 
 function(add_libhdfs_test NAME LIBRARY)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
index 846852bfd0e88..1641470733f2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
@@ -22,6 +22,7 @@
 #include "hdfspp/hdfs_ext.h"
 #include "native_mini_dfs.h"
 #include "os/thread.h"
+#include "x-platform/c-api/syscall.h"
 
 #include <errno.h>
 #include <inttypes.h>
@@ -126,7 +127,8 @@ static int hdfsCurlData(const char *host, const tPort port, const char *dirNm,
   EXPECT_NONNULL(pw = getpwuid(uid));
 
   int fd = -1;
-  EXPECT_NONNEGATIVE(fd = mkstemp(tmpFile));
+  EXPECT_NONNEGATIVE(fd = x_platform_syscall_create_and_open_temp_file(
+                         tmpFile, sizeof tmpFile));
 
   tSize sz = 0;
   while (sz < fileSz) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc
index cca9f6a024807..0bb5fc15ef631 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc
@@ -18,6 +18,26 @@
 
 #include "x-platform/syscall.h"
 
-extern "C" int x_platform_syscall_write_to_stdout(const char* msg) {
+#include <algorithm>
+#include <vector>
+
+extern "C" {
+int x_platform_syscall_write_to_stdout(const char* msg) {
   return XPlatform::Syscall::WriteToStdout(msg) ? 1 : 0;
 }
+
+int x_platform_syscall_create_and_open_temp_file(char* pattern,
+                                                 const size_t pattern_len) {
+  std::vector<char> pattern_vec(pattern, pattern + pattern_len);
+
+  const auto fd = XPlatform::Syscall::CreateAndOpenTempFile(pattern_vec);
+  if (fd != -1) {
+    std::copy_n(pattern_vec.begin(), pattern_len, pattern);
+  }
+  return fd;
+}
+
+int x_platform_syscall_close_file(const int fd) {
+  return XPlatform::Syscall::CloseFile(fd);
+}
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h
index be905ae1d364e..93878b144db98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h
@@ -24,5 +24,8 @@
  */
 
 int x_platform_syscall_write_to_stdout(const char* msg);
+int x_platform_syscall_create_and_open_temp_file(char* pattern,
+                                                 size_t pattern_len);
+int x_platform_syscall_close_file(int fd);
 
 #endif  // NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_SYSCALL_H
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
index 3a77f2dd6d026..9959f215b2068 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
@@ -20,6 +20,7 @@
 #define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_SYSCALL
 
 #include <string>
+#include <vector>
 
 /**
  * The {@link XPlatform} namespace contains components that
@@ -84,6 +85,34 @@ class Syscall {
   static bool StringCompareIgnoreCase(const std::string& a,
                                       const std::string& b);
 
+  /**
+   * Creates and opens a temporary file with a given {@link pattern}.
+   * The {@link pattern} must end with a minimum of 6 'X' characters.
+   * This function will first modify the last 6 'X' characters with
+   * random character values, which serve as the temporary file name.
+   * Subsequently opens the file and returns the file descriptor for
+   * the same. The behaviour of this function is the same as that of
+   * POSIX mkstemp function. The file must be later closed by the
+   * application and is not handled by this function.
+   *
+   * @param pattern the pattern to be used for the temporary filename.
+   * @returns an integer representing the file descriptor for the
+   * opened temporary file. Returns -1 in the case of error and sets
+   * the global errno with the appropriate error code.
+   */
+  static int CreateAndOpenTempFile(std::vector<char>& pattern);
+
+  /**
+   * Closes the file corresponding to given {@link file_descriptor}.
+   *
+   * @param file_descriptor the file descriptor of the file to close.
+   * @returns a boolean indicating the status of the call to this
+   * function. true if it's a success, false in the case of an error.
+   * The global errno is set if the call to this function was not
+   * successful.
+   */
+  static bool CloseFile(int file_descriptor);
+
  private:
   static bool WriteToStdoutImpl(const char* message);
 };
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index 06e96a8732e37..9903bbe853ba3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include <cstring>
+#include <vector>
 
 #include "syscall.h"
 
@@ -54,3 +55,13 @@ bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
                                                  const std::string& b) {
   return strcasecmp(a.c_str(), b.c_str()) == 0;
 }
+
+int XPlatform::Syscall::CreateAndOpenTempFile(std::vector<char>& pattern) {
+  // Make space for mkstemp to add NULL character at the end
+  pattern.resize(pattern.size() + 1);
+  return mkstemp(pattern.data());
+}
+
+bool XPlatform::Syscall::CloseFile(const int file_descriptor) {
+  return close(file_descriptor) == 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
index 2cd9e9d5157df..b5ddd04373841 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
@@ -19,7 +19,14 @@
 #include <Shlwapi.h>
 #include <WinBase.h>
 #include <Windows.h>
+#include <fcntl.h>
+#include <io.h>
+#include <share.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 
+#include <cerrno>
+#include <cstdlib>
 #include <cstring>
 
 #include "syscall.h"
@@ -64,3 +71,26 @@ bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
                                                  const std::string& b) {
   return _stricmp(a.c_str(), b.c_str()) == 0;
 }
+
+int XPlatform::Syscall::CreateAndOpenTempFile(std::vector<char>& pattern) {
+  if (_set_errno(0) != 0) {
+    return -1;
+  }
+
+  // Make space for _mktemp_s to add NULL character at the end
+  pattern.resize(pattern.size() + 1);
+  if (_mktemp_s(pattern.data(), pattern.size()) != 0) {
+    return -1;
+  }
+
+  auto fd{-1};
+  if (_sopen_s(&fd, pattern.data(), _O_RDWR | _O_CREAT | _O_EXCL, _SH_DENYNO,
+               _S_IREAD | _S_IWRITE) != 0) {
+    return -1;
+  }
+  return fd;
+}
+
+bool XPlatform::Syscall::CloseFile(const int file_descriptor) {
+  return _close(file_descriptor) == 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index 32f75f474b8d7..d37fcc29f9cbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -96,11 +96,13 @@ add_executable(node_exclusion_test node_exclusion_test.cc)
 target_link_libraries(node_exclusion_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(node_exclusion node_exclusion_test)
 
-add_executable(configuration_test configuration_test.cc)
+add_executable(configuration_test $<TARGET_OBJECTS:x_platform_obj> configuration_test.cc)
+target_include_directories(configuration_test PRIVATE ../lib)
 target_link_libraries(configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(configuration configuration_test)
 
-add_executable(hdfs_configuration_test hdfs_configuration_test.cc)
+add_executable(hdfs_configuration_test $<TARGET_OBJECTS:x_platform_obj> hdfs_configuration_test.cc)
+target_include_directories(hdfs_configuration_test PRIVATE ../lib)
 target_link_libraries(hdfs_configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfs_configuration hdfs_configuration_test)
 
@@ -108,11 +110,13 @@ add_executable(hdfspp_errors_test hdfspp_errors.cc)
 target_link_libraries(hdfspp_errors_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfspp_errors hdfspp_errors_test)
 
-add_executable(hdfs_builder_test hdfs_builder_test.cc)
+add_executable(hdfs_builder_test $<TARGET_OBJECTS:x_platform_obj> hdfs_builder_test.cc)
+target_include_directories(hdfs_builder_test PRIVATE ../lib)
 target_link_libraries(hdfs_builder_test test_common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfs_builder_test hdfs_builder_test)
 
 add_executable(logging_test logging_test.cc $<TARGET_OBJECTS:x_platform_obj>)
+target_include_directories(logging_test PRIVATE ../lib)
 target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(logging_test logging_test)
 
@@ -124,7 +128,8 @@ add_executable(user_lock_test user_lock_test.cc)
 target_link_libraries(user_lock_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(user_lock user_lock_test)
 
-add_executable(hdfs_config_connect_bugs_test hdfs_config_connect_bugs.cc)
+add_executable(hdfs_config_connect_bugs_test $<TARGET_OBJECTS:x_platform_obj> hdfs_config_connect_bugs.cc)
+target_include_directories(hdfs_config_connect_bugs_test PRIVATE ../lib)
 target_link_libraries(hdfs_config_connect_bugs_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfs_config_connect_bugs hdfs_config_connect_bugs_test)
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
index 9534204c92ca0..3bf2524354b6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
@@ -299,11 +299,12 @@ TEST(ConfigurationTest, TestFileReads)
   // Single stream
   {
     TempFile tempFile;
-    writeSimpleConfig(tempFile.filename, "key1", "value1");
+    writeSimpleConfig(tempFile.GetFileName(), "key1", "value1");
 
     ConfigurationLoader config_loader;
     config_loader.ClearSearchPath();
-    optional<Configuration> config = config_loader.LoadFromFile<Configuration>(tempFile.filename);
+    optional<Configuration> config =
+        config_loader.LoadFromFile<Configuration>(tempFile.GetFileName());
     EXPECT_TRUE(config && "Parse first stream");
     EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
   }
@@ -311,16 +312,18 @@ TEST(ConfigurationTest, TestFileReads)
   // Multiple files
   {
     TempFile tempFile;
-    writeSimpleConfig(tempFile.filename, "key1", "value1");
+    writeSimpleConfig(tempFile.GetFileName(), "key1", "value1");
 
     ConfigurationLoader loader;
-    optional<Configuration> config = loader.LoadFromFile<Configuration>(tempFile.filename);
+    optional<Configuration> config =
+        loader.LoadFromFile<Configuration>(tempFile.GetFileName());
     ASSERT_TRUE(config && "Parse first stream");
     EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
 
     TempFile tempFile2;
-    writeSimpleConfig(tempFile2.filename, "key2", "value2");
-    optional<Configuration> config2 = loader.OverlayResourceFile(*config, tempFile2.filename);
+    writeSimpleConfig(tempFile2.GetFileName(), "key2", "value2");
+    optional<Configuration> config2 =
+        loader.OverlayResourceFile(*config, tempFile2.GetFileName());
     ASSERT_TRUE(config2 && "Parse second stream");
     EXPECT_EQ("value1", config2->GetWithDefault("key1", ""));
     EXPECT_EQ("value2", config2->GetWithDefault("key2", ""));
@@ -350,13 +353,13 @@ TEST(ConfigurationTest, TestFileReads)
   {
     TempDir tempDir1;
     TempFile tempFile1(tempDir1.path + "/file1.xml");
-    writeSimpleConfig(tempFile1.filename, "key1", "value1");
+    writeSimpleConfig(tempFile1.GetFileName(), "key1", "value1");
     TempDir tempDir2;
     TempFile tempFile2(tempDir2.path + "/file2.xml");
-    writeSimpleConfig(tempFile2.filename, "key2", "value2");
+    writeSimpleConfig(tempFile2.GetFileName(), "key2", "value2");
     TempDir tempDir3;
     TempFile tempFile3(tempDir3.path + "/file3.xml");
-    writeSimpleConfig(tempFile3.filename, "key3", "value3");
+    writeSimpleConfig(tempFile3.GetFileName(), "key3", "value3");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir1.path + ":" + tempDir2.path + ":" + tempDir3.path);
@@ -377,7 +380,7 @@ TEST(ConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.filename, "key1", "value1");
+    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
index 9ad11b70cc8dc..23fc0d31047b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
@@ -21,11 +21,19 @@
 #include "hdfspp/config_parser.h"
 #include "common/configuration.h"
 #include "common/configuration_loader.h"
+#include "x-platform/syscall.h"
+
 #include <cstdio>
 #include <fstream>
 #include <istream>
+#include <string>
+#include <utility>
+#include <vector>
+
 #include <ftw.h>
+#include <unistd.h>
 #include <gmock/gmock.h>
+#include <gtest/gtest.h>
 
 namespace hdfs {
 
@@ -107,23 +115,51 @@ void writeDamagedConfig(const std::string& filename, Args... args) {
 
 // TempDir: is deleted on destruction
 class TempFile {
-public:
-  std::string filename;
-  char        fn_buffer[128];
-  int         tempFileHandle;
-  TempFile() : tempFileHandle(-1) {
-    strncpy(fn_buffer, "/tmp/test_XXXXXXXXXX", sizeof(fn_buffer));
-    tempFileHandle = mkstemp(fn_buffer);
-    EXPECT_NE(-1, tempFileHandle);
-    filename = fn_buffer;
+ public:
+  TempFile() {
+    std::vector<char> tmp_buf(filename_.begin(), filename_.end());
+    fd_ = XPlatform::Syscall::CreateAndOpenTempFile(tmp_buf);
+    EXPECT_NE(fd_, -1);
+    filename_.assign(tmp_buf.data());
   }
-  TempFile(const std::string & fn) : filename(fn), tempFileHandle(-1) {
-    strncpy(fn_buffer, fn.c_str(), sizeof(fn_buffer));
-    fn_buffer[sizeof(fn_buffer)-1] = 0;
+
+  TempFile(std::string fn) : filename_(std::move(fn)) {}
+
+  TempFile(const TempFile& other) = default;
+
+  TempFile(TempFile&& other) noexcept
+      : filename_{std::move(other.filename_)}, fd_{other.fd_} {}
+
+  TempFile& operator=(const TempFile& other) {
+    if (&other != this) {
+      filename_ = other.filename_;
+      fd_ = other.fd_;
+    }
+    return *this;
   }
-  ~TempFile() { if(-1 != tempFileHandle) close(tempFileHandle); unlink(fn_buffer); }
-};
 
+  TempFile& operator=(TempFile&& other) noexcept {
+    if (&other != this) {
+      filename_ = std::move(other.filename_);
+      fd_ = other.fd_;
+    }
+    return *this;
+  }
+
+  [[nodiscard]] const std::string& GetFileName() const { return filename_; }
+
+  ~TempFile() {
+    if (-1 != fd_) {
+      EXPECT_NE(XPlatform::Syscall::CloseFile(fd_), -1);
+    }
+
+    unlink(filename_.c_str());
+  }
+
+ private:
+  std::string filename_{"/tmp/test_XXXXXXXXXX"};
+  int fd_{-1};
+};
 
 // Callback to remove a directory in the nftw visitor
 int nftw_remove(const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc
index 01db69d41ef22..147cfee6be82c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc
@@ -45,7 +45,7 @@ TEST(HdfsBuilderTest, TestRead)
   {
     TempDir tempDir1;
     TempFile tempFile1(tempDir1.path + "/core-site.xml");
-    writeSimpleConfig(tempFile1.filename, "key1", "value1");
+    writeSimpleConfig(tempFile1.GetFileName(), "key1", "value1");
 
     hdfsBuilder * builder = hdfsNewBuilderFromDirectory(tempDir1.path.c_str());
 
@@ -68,7 +68,7 @@ TEST(HdfsBuilderTest, TestRead)
   {
     TempDir tempDir1;
     TempFile tempFile1(tempDir1.path + "/core-site.xml");
-    writeSimpleConfig(tempFile1.filename, "key1", "100");
+    writeSimpleConfig(tempFile1.GetFileName(), "key1", "100");
 
     hdfsBuilder * builder = hdfsNewBuilderFromDirectory(tempDir1.path.c_str());
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
index b21725c50fae7..4e1bc3b65aec9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
@@ -72,9 +72,9 @@ TEST(HdfsConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.filename, "key1", "value1");
+    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeSimpleConfig(hdfsSite.filename, "key2", "value2");
+    writeSimpleConfig(hdfsSite.GetFileName(), "key2", "value2");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
@@ -89,7 +89,7 @@ TEST(HdfsConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.filename, "key1", "value1");
+    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
@@ -103,7 +103,7 @@ TEST(HdfsConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeSimpleConfig(hdfsSite.filename, "key2", "value2");
+    writeSimpleConfig(hdfsSite.GetFileName(), "key2", "value2");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
@@ -121,9 +121,9 @@ TEST(HdfsConfigurationTest, TestConfigParserAPI) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.filename, "key1", "value1");
+    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeSimpleConfig(hdfsSite.filename, "key2", "value2");
+    writeSimpleConfig(hdfsSite.GetFileName(), "key2", "value2");
 
     ConfigParser parser(tempDir.path);
 
@@ -142,9 +142,9 @@ TEST(HdfsConfigurationTest, TestConfigParserAPI) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.filename, "key1", "value1");
+    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeDamagedConfig(hdfsSite.filename, "key2", "value2");
+    writeDamagedConfig(hdfsSite.GetFileName(), "key2", "value2");
 
     ConfigParser parser(tempDir.path);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
index 7fa3971cf7edd..a7847e1db3547 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
@@ -85,3 +85,21 @@ TEST(XPlatformSyscall, StringCompareIgnoreCaseNegative) {
   EXPECT_FALSE(XPlatform::Syscall::StringCompareIgnoreCase("abcd", "abcde"));
   EXPECT_FALSE(XPlatform::Syscall::StringCompareIgnoreCase("12345", "abcde"));
 }
+
+TEST(XPlatformSyscall, CreateAndOpenTempFileBasic) {
+  std::string pattern("tmp-XXXXXX");
+  std::vector<char> pattern_vec(pattern.begin(), pattern.end());
+
+  const auto fd = XPlatform::Syscall::CreateAndOpenTempFile(pattern_vec);
+  EXPECT_GT(fd, -1);
+  EXPECT_TRUE(XPlatform::Syscall::CloseFile(fd));
+}
+
+TEST(XPlatformSyscall, CreateAndOpenTempFileNegative) {
+  std::string pattern("does-not-adhere-to-pattern");
+  std::vector<char> pattern_vec(pattern.begin(), pattern.end());
+
+  const auto fd = XPlatform::Syscall::CreateAndOpenTempFile(pattern_vec);
+  EXPECT_EQ(fd, -1);
+  EXPECT_FALSE(XPlatform::Syscall::CloseFile(fd));
+}

From 7d6f0ca0bd2e189d200ce0f09d05b41aeb857ad4 Mon Sep 17 00:00:00 2001
From: Ayush Saxena <ayushsaxena@apache.org>
Date: Tue, 13 Apr 2021 23:28:42 +0530
Subject: [PATCH 26/43] Revert "HDFS-15884. RBF: Remove unused method
 getCreateLocation in RouterRpcServer (#2754). Contributed by tomscut."

This reverts commit e565b05c80c731898a54f8c1d358c12090ca5bbf.

The removed method needs to be used in HDFS-15423.
---
 .../server/federation/router/RouterRpcServer.java  | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 6bf159f7886a0..1d0800e4bd833 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -776,6 +776,20 @@ public HdfsFileStatus create(String src, FsPermission masked,
         replication, blockSize, supportedVersions, ecPolicyName, storagePolicy);
   }
 
+
+  /**
+   * Get the location to create a file. It checks if the file already existed
+   * in one of the locations.
+   *
+   * @param src Path of the file to check.
+   * @return The remote location for this file.
+   * @throws IOException If the file has no creation location.
+   */
+  RemoteLocation getCreateLocation(final String src) throws IOException {
+    final List<RemoteLocation> locations = getLocationsForPath(src, true);
+    return getCreateLocation(src, locations);
+  }
+
   /**
    * Get the location to create a file. It checks if the file already existed
    * in one of the locations.

From ba3bc53f4e3277618e5b9c5c440b8a997e40ed32 Mon Sep 17 00:00:00 2001
From: Miklos Gergely <mgergely@cloudera.com>
Date: Wed, 14 Apr 2021 17:16:30 +0200
Subject: [PATCH 27/43] YARN-10736. Fix GetApplicationsRequest JavaDoc.
 Contributed by Miklos Gergely. (#2906)

---
 .../yarn/api/protocolrecords/GetApplicationsRequest.java      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
index 81d98b5604885..5a00fa93f0c6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
@@ -395,7 +395,7 @@ public abstract void setStartRange(long begin, long end)
   public abstract void setScope(ApplicationsRequestScope scope);
 
   /**
-   * Set the name to filter applications.
+   * Get the name to filter applications.
    *
    * @return the name
    */
@@ -404,7 +404,7 @@ public abstract void setStartRange(long begin, long end)
   public abstract String getName();
 
   /**
-   * Get the name to filter applications.
+   * Set the name to filter applications.
    *
    * @param name of the application
    */

From 9179638017439ce08c4de8414361304628eb64f4 Mon Sep 17 00:00:00 2001
From: Viraj Jasani <vjasani@apache.org>
Date: Thu, 15 Apr 2021 14:34:46 +0530
Subject: [PATCH 28/43] HADOOP-17524. Remove EventCounter and Log counters from
 JVM Metrics (#2909)

Reviewed-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
---
 .../org/apache/hadoop/log/EventCounter.java   |  34 ------
 .../hadoop/log/metrics/EventCounter.java      | 100 ------------------
 .../hadoop/metrics2/source/JvmMetrics.java    |   8 --
 .../metrics2/source/TestJvmMetrics.java       |   5 -
 4 files changed, 147 deletions(-)
 delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/EventCounter.java
 delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/metrics/EventCounter.java

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/EventCounter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/EventCounter.java
deleted file mode 100644
index 3192d06807326..0000000000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/EventCounter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.log;
-
-/**
- * A log4J Appender that simply counts logging events in three levels:
- * fatal, error and warn. The class name is used in log4j.properties
- * @deprecated use {@link org.apache.hadoop.log.metrics.EventCounter} instead
- */
-@Deprecated
-public class EventCounter extends org.apache.hadoop.log.metrics.EventCounter {
-  static {
-    // The logging system is not started yet.
-    System.err.println("WARNING: "+ EventCounter.class.getName() +
-        " is deprecated. Please use "+
-        org.apache.hadoop.log.metrics.EventCounter.class.getName() +
-        " in all the log4j.properties files.");
-  }
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/metrics/EventCounter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/metrics/EventCounter.java
deleted file mode 100644
index 95b9cfc5409b0..0000000000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/metrics/EventCounter.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.log.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Level;
-import org.apache.log4j.spi.LoggingEvent;
-
-/**
- * A log4J Appender that simply counts logging events in three levels:
- * fatal, error and warn. The class name is used in log4j.properties
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class EventCounter extends AppenderSkeleton {
-  private static final int FATAL = 0;
-  private static final int ERROR = 1;
-  private static final int WARN = 2;
-  private static final int INFO = 3;
-
-  private static class EventCounts {
-    private final long[] counts = {0, 0, 0, 0};
-
-    private synchronized void incr(int i) {
-      ++counts[i];
-    }
-
-    private synchronized long get(int i) {
-      return counts[i];
-    }
-  }
-
-  private static EventCounts counts = new EventCounts();
-
-  @InterfaceAudience.Private
-  public static long getFatal() {
-    return counts.get(FATAL);
-  }
-
-  @InterfaceAudience.Private
-  public static long getError() {
-    return counts.get(ERROR);
-  }
-
-  @InterfaceAudience.Private
-  public static long getWarn() {
-    return counts.get(WARN);
-  }
-
-  @InterfaceAudience.Private
-  public static long getInfo() {
-    return counts.get(INFO);
-  }
-
-  @Override
-  public void append(LoggingEvent event) {
-    Level level = event.getLevel();
-    // depends on the api, == might not work
-    // see HADOOP-7055 for details
-    if (level.equals(Level.INFO)) {
-      counts.incr(INFO);
-    }
-    else if (level.equals(Level.WARN)) {
-      counts.incr(WARN);
-    }
-    else if (level.equals(Level.ERROR)) {
-      counts.incr(ERROR);
-    }
-    else if (level.equals(Level.FATAL)) {
-      counts.incr(FATAL);
-    }
-  }
-
-  @Override
-  public void close() {
-  }
-
-  @Override
-  public boolean requiresLayout() {
-    return false;
-  }
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index 816940b109879..9e5a78fd90571 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -33,7 +33,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.log.metrics.EventCounter;
 import org.apache.hadoop.metrics2.MetricsCollector;
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -154,7 +153,6 @@ public void getMetrics(MetricsCollector collector, boolean all) {
     } else {
       getThreadUsageFromGroup(rb);
     }
-    getEventCounters(rb);
   }
 
   private void getMemoryUsage(MetricsRecordBuilder rb) {
@@ -284,10 +282,4 @@ private void getThreadUsageFromGroup(MetricsRecordBuilder rb) {
         .addGauge(ThreadsTerminated, threadsTerminated);
   }
 
-  private void getEventCounters(MetricsRecordBuilder rb) {
-    rb.addCounter(LogFatal, EventCounter.getFatal())
-      .addCounter(LogError, EventCounter.getError())
-      .addCounter(LogWarn, EventCounter.getWarn())
-      .addCounter(LogInfo, EventCounter.getInfo());
-  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
index 6fdd64dca7c30..ea86fc14c7978 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
@@ -78,13 +78,8 @@ public void testJvmPauseMonitorPresence() {
     for (JvmMetricsInfo info : JvmMetricsInfo.values()) {
       if (info.name().startsWith("Mem")) {
         verify(rb).addGauge(eq(info), anyFloat());
-      } else if (info.name().startsWith("Gc") &&
-          !info.name().equals("GcTimePercentage")) {
-        verify(rb).addCounter(eq(info), anyLong());
       } else if (info.name().startsWith("Threads")) {
         verify(rb).addGauge(eq(info), anyInt());
-      } else if (info.name().startsWith("Log")) {
-        verify(rb).addCounter(eq(info), anyLong());
       }
     }
   }

From 2717203f858ff654de0fc01cfb9afef2e705e33c Mon Sep 17 00:00:00 2001
From: Masatake Iwasaki <iwasakims@apache.org>
Date: Thu, 15 Apr 2021 21:33:18 +0900
Subject: [PATCH 29/43] HADOOP-17569. Building native code fails on Fedora 33.
 (#2886)

---
 .../hadoop-common/src/main/native/src/exception.c             | 4 ++--
 .../hadoop-hdfs-native-client/src/CMakeLists.txt              | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/exception.c b/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
index fc072e8002bf2..a25cc3d3b7eef 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
@@ -111,8 +111,8 @@ jthrowable newIOException(JNIEnv* env, const char *fmt, ...)
 const char* terror(int errnum)
 {
 
-#if defined(__sun)
-// MT-Safe under Solaris which doesn't support sys_errlist/sys_nerr
+#if defined(__sun) || defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 32)
+// MT-Safe under Solaris or glibc >= 2.32 not supporting sys_errlist/sys_nerr
   return strerror(errnum); 
 #else
   if ((errnum < 0) || (errnum >= sys_nerr)) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index 4e943de1773f2..d8c2012fb88de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -69,6 +69,8 @@ if(WIN32)
     set(OUT_DIR bin)
 else()
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
+    # using old default behavior on GCC >= 10.0
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fcommon")
     set(OS_DIR ${CMAKE_SOURCE_DIR}/main/native/libhdfs/os/posix)
 
     # IMPORTANT: OUT_DIR MUST be relative to maven's

From c8210088365fd26eaacc02f6f8b03d93149183bd Mon Sep 17 00:00:00 2001
From: Vivek Ratnavel Subramanian <vivekratnavel90@gmail.com>
Date: Thu, 15 Apr 2021 14:45:51 -0700
Subject: [PATCH 30/43] HDFS-15850. Superuser actions should be reported to
 external enforcers (#2784)

---
 .../ContentSummaryComputationContext.java     |  9 ++-
 .../hdfs/server/namenode/FSDirAttrOp.java     | 35 ++++++---
 .../namenode/FSDirStatAndListingOp.java       |  7 +-
 .../hdfs/server/namenode/FSDirXAttrOp.java    |  5 +-
 .../hdfs/server/namenode/FSDirectory.java     | 30 +++++---
 .../hdfs/server/namenode/FSNamesystem.java    | 74 ++++++++++--------
 .../server/namenode/FSPermissionChecker.java  | 75 +++++++++++++++++--
 .../namenode/INodeAttributeProvider.java      | 45 ++++++++++-
 .../hadoop/hdfs/server/namenode/NameNode.java | 24 +++---
 .../server/namenode/NameNodeRpcServer.java    | 51 ++++++++-----
 .../hdfs/server/namenode/NamenodeFsck.java    |  3 +-
 .../namenode/XAttrPermissionFilter.java       | 16 +++-
 12 files changed, 270 insertions(+), 104 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index 7a5963a6c57cd..e304baf652843 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -205,8 +205,13 @@ public String getErasureCodingPolicyName(INode inode) {
   void checkPermission(INodeDirectory inode, int snapshotId, FsAction access)
       throws AccessControlException {
     if (dir != null && dir.isPermissionEnabled()
-        && pc != null && !pc.isSuperUser()) {
-      pc.checkPermission(inode, snapshotId, access);
+        && pc != null) {
+      if (pc.isSuperUser()) {
+        // call external enforcer for audit
+        pc.checkSuperuserPrivilege(inode.getFullPathName());
+      } else {
+        pc.checkPermission(inode, snapshotId, access);
+      }
     }
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 173348f356473..5914d7449619c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -83,14 +83,25 @@ static FileStatus setOwner(
     try {
       iip = fsd.resolvePath(pc, src, DirOp.WRITE);
       fsd.checkOwner(pc, iip);
-      if (!pc.isSuperUser()) {
-        if (username != null && !pc.getUser().equals(username)) {
-          throw new AccessControlException("User " + pc.getUser()
-              + " is not a super user (non-super user cannot change owner).");
-        }
-        if (group != null && !pc.isMemberOfGroup(group)) {
-          throw new AccessControlException(
-              "User " + pc.getUser() + " does not belong to " + group);
+      // At this point, the user must be either owner or super user.
+      // superuser: can change owner to a different user,
+      // change owner group to any group
+      // owner: can't change owner to a different user but can change owner
+      // group to different group that the user belongs to.
+      if ((username != null && !pc.getUser().equals(username)) ||
+          (group != null && !pc.isMemberOfGroup(group))) {
+        try {
+          // check if the user is superuser
+          pc.checkSuperuserPrivilege(iip.getPath());
+        } catch (AccessControlException e) {
+          if (username != null && !pc.getUser().equals(username)) {
+            throw new AccessControlException("User " + pc.getUser()
+                + " is not a super user (non-super user cannot change owner).");
+          }
+          if (group != null && !pc.isMemberOfGroup(group)) {
+            throw new AccessControlException(
+                "User " + pc.getUser() + " does not belong to " + group);
+          }
         }
       }
       changed = unprotectedSetOwner(fsd, iip, username, group);
@@ -238,10 +249,12 @@ static void setQuota(FSDirectory fsd, FSPermissionChecker pc, String src,
     fsd.writeLock();
     try {
       INodesInPath iip = fsd.resolvePath(pc, src, DirOp.WRITE);
+      // Here, the assumption is that the caller of this method has
+      // already checked for super user privilege
       if (fsd.isPermissionEnabled() && !pc.isSuperUser() && allowOwner) {
-        INodeDirectory parentDir= iip.getLastINode().getParent();
-        if (parentDir == null ||
-            !parentDir.getUserName().equals(pc.getUser())) {
+        try {
+          fsd.checkOwner(pc, iip.getParentINodesInPath());
+        } catch(AccessControlException ace) {
           throw new AccessControlException(
               "Access denied for user " + pc.getUser() +
               ". Superuser or owner of parent folder privilege is required");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index dfacc491eae53..8aff179358896 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -105,6 +105,7 @@ static HdfsFileStatus getFileInfo(FSDirectory fsd, FSPermissionChecker pc,
       // superuser to receive null instead.
       try {
         iip = fsd.resolvePath(pc, srcArg, dirOp);
+        pc.checkSuperuserPrivilege(iip.getPath());
       } catch (AccessControlException ace) {
         return null;
       }
@@ -151,12 +152,14 @@ static GetBlockLocationsResult getBlockLocations(
     BlockManager bm = fsd.getBlockManager();
     fsd.readLock();
     try {
-      final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.READ);
+      // Just get INodesInPath without access checks, since we check for path
+      // access later
+      final INodesInPath iip = fsd.resolvePath(null, src, DirOp.READ);
       src = iip.getPath();
       final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
       if (fsd.isPermissionEnabled()) {
-        fsd.checkPathAccess(pc, iip, FsAction.READ);
         fsd.checkUnreadableBySuperuser(pc, iip);
+        fsd.checkPathAccess(pc, iip, FsAction.READ);
       }
 
       final long fileSize = iip.isSnapshot()
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index ef83fe982b24c..ce79321f96801 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -437,7 +437,10 @@ private static void checkXAttrChangeAccess(
       if (inode != null &&
           inode.isDirectory() &&
           inode.getFsPermission().getStickyBit()) {
-        if (!pc.isSuperUser()) {
+        if (pc.isSuperUser()) {
+          // call external enforcer for audit
+          pc.checkSuperuserPrivilege(iip.getPath());
+        } else {
           fsd.checkOwner(pc, iip);
         }
       } else {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 0e15921ba4953..497aa84e767a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -717,18 +717,18 @@ public INodesInPath resolvePath(FSPermissionChecker pc, String src,
 
     byte[][] components = INode.getPathComponents(src);
     boolean isRaw = isReservedRawName(components);
+    components = resolveComponents(components, this);
+    INodesInPath iip = INodesInPath.resolve(rootDir, components, isRaw);
     if (isPermissionEnabled && pc != null && isRaw) {
       switch(dirOp) {
-        case READ_LINK:
-        case READ:
-          break;
-        default:
-          pc.checkSuperuserPrivilege();
-          break;
+      case READ_LINK:
+      case READ:
+        break;
+      default:
+        pc.checkSuperuserPrivilege(iip.getPath());
+        break;
       }
     }
-    components = resolveComponents(components, this);
-    INodesInPath iip = INodesInPath.resolve(rootDir, components, isRaw);
     // verify all ancestors are dirs and traversable.  note that only
     // methods that create new namespace items have the signature to throw
     // PNDE
@@ -1942,7 +1942,10 @@ void checkPermission(FSPermissionChecker pc, INodesInPath iip,
       boolean doCheckOwner, FsAction ancestorAccess, FsAction parentAccess,
       FsAction access, FsAction subAccess, boolean ignoreEmptyDir)
       throws AccessControlException {
-    if (!pc.isSuperUser()) {
+    if (pc.isSuperUser()) {
+      // call the external enforcer for audit
+      pc.checkSuperuserPrivilege(iip.getPath());
+    } else {
       readLock();
       try {
         pc.checkPermission(iip, doCheckOwner, ancestorAccess,
@@ -1958,9 +1961,12 @@ void checkUnreadableBySuperuser(FSPermissionChecker pc, INodesInPath iip)
     if (pc.isSuperUser()) {
       if (FSDirXAttrOp.getXAttrByPrefixedName(this, iip,
           SECURITY_XATTR_UNREADABLE_BY_SUPERUSER) != null) {
-        throw new AccessControlException(
-            "Access is denied for " + pc.getUser() + " since the superuser "
-            + "is not allowed to perform this operation.");
+        String errorMessage = "Access is denied for " + pc.getUser()
+            + " since the superuser is not allowed to perform this operation.";
+        pc.denyUserAccess(iip.getPath(), errorMessage);
+      } else {
+        // call the external enforcer for audit.
+        pc.checkSuperuserPrivilege(iip.getPath());
       }
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7d9f78c0647fb..e559515696d66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1954,7 +1954,7 @@ BatchedListEntries<OpenFileEntry> listOpenFiles(long prevId,
       EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException {
     INode.checkAbsolutePath(path);
     final String operationName = "listOpenFiles";
-    checkSuperuserPrivilege();
+    checkSuperuserPrivilege(operationName, path);
     checkOperation(OperationCategory.READ);
     BatchedListEntries<OpenFileEntry> batchedListEntries;
     String normalizedPath = new Path(path).toString(); // normalize path.
@@ -2289,6 +2289,7 @@ boolean truncate(String src, long newLength, String clientName,
     final String operationName = "truncate";
     requireEffectiveLayoutVersionForFeature(Feature.TRUNCATE);
     FSDirTruncateOp.TruncateResult r = null;
+    FileStatus status;
     try {
       NameNode.stateChangeLog.info(
           "DIR* NameSystem.truncate: src={} newLength={}", src, newLength);
@@ -2307,7 +2308,7 @@ boolean truncate(String src, long newLength, String clientName,
         r = FSDirTruncateOp.truncate(this, src, newLength, clientName,
             clientMachine, mtime, toRemoveBlocks, pc);
       } finally {
-        FileStatus status = r != null ? r.getFileStatus() : null;
+        status = r != null ? r.getFileStatus() : null;
         writeUnlock(operationName,
             getLockReportInfoSupplier(src, null, status));
       }
@@ -2316,11 +2317,12 @@ boolean truncate(String src, long newLength, String clientName,
         removeBlocks(toRemoveBlocks);
         toRemoveBlocks.clear();
       }
-      logAuditEvent(true, operationName, src, null, r.getFileStatus());
+      logAuditEvent(true, operationName, src, null, status);
     } catch (AccessControlException e) {
       logAuditEvent(false, operationName, src);
       throw e;
     }
+    assert(r != null);
     return r.getResult();
   }
 
@@ -3582,7 +3584,7 @@ void setQuota(String src, long nsQuota, long ssQuota, StorageType type)
     FSPermissionChecker.setOperationType(operationName);
     try {
       if(!allowOwnerSetQuota) {
-        checkSuperuserPrivilege(pc);
+        checkSuperuserPrivilege(operationName, src);
       }
       writeLock();
       try {
@@ -5222,18 +5224,22 @@ PermissionStatus createFsOwnerPermissions(FsPermission permission) {
     return new PermissionStatus(fsOwner.getShortUserName(), supergroup, permission);
   }
 
+  /**
+   * This method is retained for backward compatibility.
+   * Please use {@link #checkSuperuserPrivilege(String)} instead.
+   *
+   * @throws AccessControlException if user is not a super user.
+   */
   void checkSuperuserPrivilege() throws AccessControlException {
     if (isPermissionEnabled) {
       FSPermissionChecker pc = getPermissionChecker();
-      pc.checkSuperuserPrivilege();
+      pc.checkSuperuserPrivilege(null);
     }
   }
 
-  void checkSuperuserPrivilege(FSPermissionChecker pc)
-      throws AccessControlException {
-    if (isPermissionEnabled) {
-      pc.checkSuperuserPrivilege();
-    }
+  void checkSuperuserPrivilege(String operationName)
+      throws IOException {
+    checkSuperuserPrivilege(operationName, null);
   }
 
   /**
@@ -6011,7 +6017,8 @@ public String toString() {
    */
   Collection<CorruptFileBlockInfo> listCorruptFileBlocks(String path,
   String[] cookieTab) throws IOException {
-    checkSuperuserPrivilege();
+    final String operationName = "listCorruptFileBlocks";
+    checkSuperuserPrivilege(operationName, path);
     checkOperation(OperationCategory.READ);
 
     int count = 0;
@@ -6939,7 +6946,7 @@ public SnapshotManager getSnapshotManager() {
   void allowSnapshot(String path) throws IOException {
     checkOperation(OperationCategory.WRITE);
     final String operationName = "allowSnapshot";
-    checkSuperuserPrivilege(operationName);
+    checkSuperuserPrivilege(operationName, path);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
@@ -6956,7 +6963,7 @@ void allowSnapshot(String path) throws IOException {
   void disallowSnapshot(String path) throws IOException {
     checkOperation(OperationCategory.WRITE);
     final String operationName = "disallowSnapshot";
-    checkSuperuserPrivilege(operationName);
+    checkSuperuserPrivilege(operationName, path);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
@@ -7666,13 +7673,14 @@ void addCachePool(CachePoolInfo req, boolean logRetryCache)
     final String operationName = "addCachePool";
     checkOperation(OperationCategory.WRITE);
     String poolInfoStr = null;
+    String poolName = req == null ? null : req.getPoolName();
     try {
-      checkSuperuserPrivilege();
+      checkSuperuserPrivilege(operationName, poolName);
       writeLock();
       try {
         checkOperation(OperationCategory.WRITE);
         checkNameNodeSafeMode("Cannot add cache pool"
-            + (req == null ? null : req.getPoolName()));
+            + poolName);
         CachePoolInfo info = FSNDNCacheOp.addCachePool(this, cacheManager, req,
             logRetryCache);
         poolInfoStr = info.toString();
@@ -7694,7 +7702,7 @@ void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
     String poolNameStr = "{poolName: " +
         (req == null ? null : req.getPoolName()) + "}";
     try {
-      checkSuperuserPrivilege();
+      checkSuperuserPrivilege(operationName, poolNameStr);
       writeLock();
       try {
         checkOperation(OperationCategory.WRITE);
@@ -7721,7 +7729,7 @@ void removeCachePool(String cachePoolName, boolean logRetryCache)
     checkOperation(OperationCategory.WRITE);
     String poolNameStr = "{poolName: " + cachePoolName + "}";
     try {
-      checkSuperuserPrivilege();
+      checkSuperuserPrivilege(operationName, poolNameStr);
       writeLock();
       try {
         checkOperation(OperationCategory.WRITE);
@@ -7926,8 +7934,7 @@ void createEncryptionZone(final String src, final String keyName,
       Metadata metadata = FSDirEncryptionZoneOp.ensureKeyIsInitialized(dir,
           keyName, src);
       final FSPermissionChecker pc = getPermissionChecker();
-      FSPermissionChecker.setOperationType(operationName);
-      checkSuperuserPrivilege(pc);
+      checkSuperuserPrivilege(operationName, src);
       checkOperation(OperationCategory.WRITE);
       writeLock();
       try {
@@ -7988,9 +7995,7 @@ BatchedListEntries<EncryptionZone> listEncryptionZones(long prevId)
     final String operationName = "listEncryptionZones";
     boolean success = false;
     checkOperation(OperationCategory.READ);
-    final FSPermissionChecker pc = getPermissionChecker();
-    FSPermissionChecker.setOperationType(operationName);
-    checkSuperuserPrivilege(pc);
+    checkSuperuserPrivilege(operationName, dir.rootDir.getFullPathName());
     readLock();
     try {
       checkOperation(OperationCategory.READ);
@@ -8006,12 +8011,13 @@ BatchedListEntries<EncryptionZone> listEncryptionZones(long prevId)
 
   void reencryptEncryptionZone(final String zone, final ReencryptAction action,
       final boolean logRetryCache) throws IOException {
+    final String operationName = "reencryptEncryptionZone";
     boolean success = false;
     try {
       Preconditions.checkNotNull(zone, "zone is null.");
       checkOperation(OperationCategory.WRITE);
       final FSPermissionChecker pc = dir.getPermissionChecker();
-      checkSuperuserPrivilege(pc);
+      checkSuperuserPrivilege(operationName, zone);
       checkNameNodeSafeMode("NameNode in safemode, cannot " + action
           + " re-encryption on zone " + zone);
       reencryptEncryptionZoneInt(pc, zone, action, logRetryCache);
@@ -8026,9 +8032,7 @@ BatchedListEntries<ZoneReencryptionStatus> listReencryptionStatus(
     final String operationName = "listReencryptionStatus";
     boolean success = false;
     checkOperation(OperationCategory.READ);
-    final FSPermissionChecker pc = getPermissionChecker();
-    FSPermissionChecker.setOperationType(operationName);
-    checkSuperuserPrivilege(pc);
+    checkSuperuserPrivilege(operationName, dir.rootDir.getFullPathName());
     readLock();
     try {
       checkOperation(OperationCategory.READ);
@@ -8871,15 +8875,19 @@ private ECTopologyVerifierResult getEcTopologyVerifierResultForEnabledPolicies()
             Arrays.asList(enabledEcPolicies));
   }
 
-  // This method logs operatoinName without super user privilege.
+  // This method logs operationName without super user privilege.
   // It should be called without holding FSN lock.
-  void checkSuperuserPrivilege(String operationName)
+  void checkSuperuserPrivilege(String operationName, String path)
       throws IOException {
-    try {
-      checkSuperuserPrivilege();
-    } catch (AccessControlException ace) {
-      logAuditEvent(false, operationName, null);
-      throw ace;
+    if (isPermissionEnabled) {
+      try {
+        FSPermissionChecker.setOperationType(operationName);
+        FSPermissionChecker pc = getPermissionChecker();
+        pc.checkSuperuserPrivilege(path);
+      } catch(AccessControlException ace){
+        logAuditEvent(false, operationName, path);
+        throw ace;
+      }
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index 3f80952a8e9af..e8e292761d40c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
 import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer;
+import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AuthorizationContext;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -144,18 +145,74 @@ private AccessControlEnforcer getAccessControlEnforcer() {
         ? attributeProvider.getExternalAccessControlEnforcer(this) : this;
   }
 
+  private AuthorizationContext getAuthorizationContextForSuperUser(
+      String path) {
+    String opType = operationType.get();
+
+    AuthorizationContext.Builder builder =
+        new INodeAttributeProvider.AuthorizationContext.Builder();
+    builder.fsOwner(fsOwner).
+        supergroup(supergroup).
+        callerUgi(callerUgi).
+        operationName(opType).
+        callerContext(CallerContext.getCurrent());
+
+    // Add path to the context builder only if it is not null.
+    if (path != null && !path.isEmpty()) {
+      builder.path(path);
+    }
+
+    return builder.build();
+  }
+
+  /**
+   * This method is retained to maintain backward compatibility.
+   * Please use the new method {@link #checkSuperuserPrivilege(String)} to make
+   * sure that the external enforcers have the correct context to audit.
+   *
+   * @throws AccessControlException if the caller is not a super user.
+   */
+  public void checkSuperuserPrivilege() throws AccessControlException {
+    checkSuperuserPrivilege(null);
+  }
+
   /**
-   * Verify if the caller has the required permission. This will result into 
-   * an exception if the caller is not allowed to access the resource.
+   * Checks if the caller has super user privileges.
+   * Throws {@link AccessControlException} for non super users.
+   *
+   * @param path The resource path for which permission is being requested.
+   * @throws AccessControlException if the caller is not a super user.
+   */
+  public void checkSuperuserPrivilege(String path)
+      throws AccessControlException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("SUPERUSER ACCESS CHECK: " + this
+          + ", operationName=" + FSPermissionChecker.operationType.get()
+          + ", path=" + path);
+    }
+    getAccessControlEnforcer().checkSuperUserPermissionWithContext(
+        getAuthorizationContextForSuperUser(path));
+  }
+
+  /**
+   * Calls the external enforcer to notify denial of access to the user with
+   * the given error message. Always throws an ACE with the given message.
+   *
+   * @param path The resource path for which permission is being requested.
+   * @param errorMessage message for the exception.
+   * @throws AccessControlException with the error message.
    */
-  public void checkSuperuserPrivilege()
+  public void denyUserAccess(String path, String errorMessage)
       throws AccessControlException {
-    if (!isSuperUser()) {
-      throw new AccessControlException("Access denied for user " 
-          + getUser() + ". Superuser privilege is required");
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("DENY USER ACCESS: " + this
+          + ", operationName=" + FSPermissionChecker.operationType.get()
+          + ", path=" + path);
     }
+    getAccessControlEnforcer().denyUserAccess(
+        getAuthorizationContextForSuperUser(path), errorMessage);
   }
-  
+
   /**
    * Check whether current user have permissions to access the path.
    * Traverse is always checked.
@@ -705,6 +762,10 @@ static void checkTraverse(FSPermissionChecker pc, INodesInPath iip,
           UnresolvedPathException, ParentNotDirectoryException {
     try {
       if (pc == null || pc.isSuperUser()) {
+        if (pc != null) {
+          // call the external enforcer for audit
+          pc.checkSuperuserPrivilege(iip.getPath());
+        }
         checkSimpleTraverse(iip);
       } else {
         pc.checkPermission(iip, false, null, null, null, null, false);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java
index e83c962a4a845..f5361adc4cff4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java
@@ -362,7 +362,7 @@ public interface AccessControlEnforcer {
      * Checks permission on a file system object. Has to throw an Exception
      * if the filesystem object is not accessible by the calling Ugi.
      * @param fsOwner Filesystem owner (The Namenode user)
-     * @param supergroup super user geoup
+     * @param supergroup super user group
      * @param callerUgi UserGroupInformation of the caller
      * @param inodeAttrs Array of INode attributes for each path element in the
      *                   the path
@@ -393,7 +393,7 @@ public abstract void checkPermission(String fsOwner, String supergroup,
 
     /**
      * Checks permission on a file system object. Has to throw an Exception
-     * if the filesystem object is not accessessible by the calling Ugi.
+     * if the filesystem object is not accessible by the calling Ugi.
      * @param authzContext an {@link AuthorizationContext} object encapsulating
      *                     the various parameters required to authorize an
      *                     operation.
@@ -405,7 +405,48 @@ default void checkPermissionWithContext(AuthorizationContext authzContext)
           + "implement the checkPermissionWithContext(AuthorizationContext) "
           + "API.");
     }
+
+    /**
+     * Checks if the user is a superuser or belongs to superuser group.
+     * It throws an AccessControlException if user is not a superuser.
+     *
+     * @param authzContext an {@link AuthorizationContext} object encapsulating
+     *                     the various parameters required to authorize an
+     *                     operation.
+     * @throws AccessControlException - if user is not a super user or part
+     * of the super user group.
+     */
+    default void checkSuperUserPermissionWithContext(
+        AuthorizationContext authzContext)
+        throws AccessControlException {
+      UserGroupInformation callerUgi = authzContext.getCallerUgi();
+      boolean isSuperUser =
+          callerUgi.getShortUserName().equals(authzContext.getFsOwner()) ||
+          callerUgi.getGroupsSet().contains(authzContext.getSupergroup());
+      if (!isSuperUser) {
+        throw new AccessControlException("Access denied for user " +
+            callerUgi.getShortUserName() + ". Superuser privilege is " +
+            "required for operation " + authzContext.getOperationName());
+      }
+    }
+
+    /**
+     * This method must be called when denying access to users to
+     * notify the external enforcers.
+     * This will help the external enforcers to audit the requests
+     * by users that were denied access.
+     * @param authzContext an {@link AuthorizationContext} object encapsulating
+     *                     the various parameters required to authorize an
+     *                     operation.
+     * @throws AccessControlException
+     */
+    default void denyUserAccess(AuthorizationContext authzContext,
+                                String errorMessage)
+        throws AccessControlException {
+      throw new AccessControlException(errorMessage);
+    }
   }
+
   /**
    * Initialize the provider. This method is called at NameNode startup
    * time.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 55196c4d44f03..ed3d80bbfc201 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -1835,9 +1835,9 @@ public static void main(String argv[]) throws Exception {
     }
   }
 
-  synchronized void monitorHealth() 
-      throws HealthCheckFailedException, AccessControlException {
-    namesystem.checkSuperuserPrivilege();
+  synchronized void monitorHealth() throws IOException {
+    String operationName = "monitorHealth";
+    namesystem.checkSuperuserPrivilege(operationName);
     if (!haEnabled) {
       return; // no-op, if HA is not enabled
     }
@@ -1859,9 +1859,9 @@ synchronized void monitorHealth()
     }
   }
   
-  synchronized void transitionToActive() 
-      throws ServiceFailedException, AccessControlException {
-    namesystem.checkSuperuserPrivilege();
+  synchronized void transitionToActive() throws IOException {
+    String operationName = "transitionToActive";
+    namesystem.checkSuperuserPrivilege(operationName);
     if (!haEnabled) {
       throw new ServiceFailedException("HA for namenode is not enabled");
     }
@@ -1876,18 +1876,18 @@ synchronized void transitionToActive()
     state.setState(haContext, ACTIVE_STATE);
   }
 
-  synchronized void transitionToStandby()
-      throws ServiceFailedException, AccessControlException {
-    namesystem.checkSuperuserPrivilege();
+  synchronized void transitionToStandby() throws IOException {
+    String operationName = "transitionToStandby";
+    namesystem.checkSuperuserPrivilege(operationName);
     if (!haEnabled) {
       throw new ServiceFailedException("HA for namenode is not enabled");
     }
     state.setState(haContext, STANDBY_STATE);
   }
 
-  synchronized void transitionToObserver()
-      throws ServiceFailedException, AccessControlException {
-    namesystem.checkSuperuserPrivilege();
+  synchronized void transitionToObserver() throws IOException {
+    String operationName = "transitionToObserver";
+    namesystem.checkSuperuserPrivilege(operationName);
     if (!haEnabled) {
       throw new ServiceFailedException("HA for namenode is not enabled");
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index aee4f68bdc7f2..90819c28ffc3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -642,6 +642,7 @@ private static UserGroupInformation getRemoteUser() throws IOException {
   public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long
       minBlockSize, long timeInterval)
       throws IOException {
+    String operationName = "getBlocks";
     if(size <= 0) {
       throw new IllegalArgumentException(
           "Unexpected not positive size: "+size);
@@ -651,15 +652,16 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long
           "Unexpected not positive size: "+size);
     }
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     namesystem.checkNameNodeSafeMode("Cannot execute getBlocks");
     return namesystem.getBlocks(datanode, size, minBlockSize, timeInterval);
   }
 
   @Override // NamenodeProtocol
   public ExportedBlockKeys getBlockKeys() throws IOException {
+    String operationName = "getBlockKeys";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     return namesystem.getBlockManager().getBlockKeys();
   }
 
@@ -667,9 +669,10 @@ public ExportedBlockKeys getBlockKeys() throws IOException {
   public void errorReport(NamenodeRegistration registration,
                           int errorCode, 
                           String msg) throws IOException {
+    String operationName = "errorReport";
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.UNCHECKED);
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     verifyRequest(registration);
     LOG.info("Error report from " + registration + ": " + msg);
     if (errorCode == FATAL) {
@@ -680,8 +683,9 @@ public void errorReport(NamenodeRegistration registration,
   @Override // NamenodeProtocol
   public NamenodeRegistration registerSubordinateNamenode(
       NamenodeRegistration registration) throws IOException {
+    String operationName = "registerSubordinateNamenode";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     verifyLayoutVersion(registration.getVersion());
     NamenodeRegistration myRegistration = nn.setRegistration();
     namesystem.registerBackupNode(registration, myRegistration);
@@ -691,8 +695,9 @@ public NamenodeRegistration registerSubordinateNamenode(
   @Override // NamenodeProtocol
   public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
       throws IOException {
+    String operationName = "startCheckpoint";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     verifyRequest(registration);
     if(!nn.isRole(NamenodeRole.NAMENODE))
       throw new IOException("Only an ACTIVE node can invoke startCheckpoint.");
@@ -714,8 +719,9 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
   @Override // NamenodeProtocol
   public void endCheckpoint(NamenodeRegistration registration,
                             CheckpointSignature sig) throws IOException {
+    String operationName = "endCheckpoint";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1322,17 +1328,19 @@ public void refreshNodes() throws IOException {
 
   @Override // NamenodeProtocol
   public long getTransactionID() throws IOException {
+    String operationName = "getTransactionID";
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.UNCHECKED);
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     return namesystem.getFSImage().getCorrectLastAppliedOrWrittenTxId();
   }
   
   @Override // NamenodeProtocol
   public long getMostRecentCheckpointTxId() throws IOException {
+    String operationName = "getMostRecentCheckpointTxId";
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.UNCHECKED);
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     return namesystem.getFSImage().getMostRecentCheckpointTxId();
   }
   
@@ -1345,23 +1353,26 @@ public CheckpointSignature rollEditLog() throws IOException {
   @Override // NamenodeProtocol
   public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
       throws IOException {
+    String operationName = "getEditLogManifest";
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.READ);
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     return namesystem.getEditLog().getEditLogManifest(sinceTxId);
   }
 
   @Override // NamenodeProtocol
   public boolean isUpgradeFinalized() throws IOException {
+    String operationName = "isUpgradeFinalized";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     return namesystem.isUpgradeFinalized();
   }
 
   @Override // NamenodeProtocol
   public boolean isRollingUpgrade() throws IOException {
+    String operationName = "isRollingUpgrade";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     return namesystem.isRollingUpgrade();
   }
     
@@ -2345,9 +2356,10 @@ public void checkAccess(String path, FsAction mode) throws IOException {
 
   @Override // ClientProtocol
   public long getCurrentEditLogTxid() throws IOException {
+    String operationName = "getCurrentEditLogTxid";
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.READ); // only active
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     // if it's not yet open for write, we may be in the process of transitioning
     // from standby to active and may not yet know what the latest committed
     // txid is
@@ -2374,9 +2386,10 @@ private static FSEditLogOp readOp(EditLogInputStream elis)
 
   @Override // ClientProtocol
   public EventBatchList getEditsFromTxid(long txid) throws IOException {
+    String operationName = "getEditsFromTxid";
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.READ); // only active
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     int maxEventsPerRPC = nn.getConf().getInt(
         DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY,
         DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT);
@@ -2521,8 +2534,9 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies(
   @Override
   public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
       ErasureCodingPolicy[] policies) throws IOException {
+    String operationName = "addErasureCodingPolicies";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     final CacheEntryWithPayload cacheEntry =
         RetryCache.waitForCompletion(retryCache, null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -2544,8 +2558,9 @@ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
   @Override
   public void removeErasureCodingPolicy(String ecPolicyName)
       throws IOException {
+    String operationName = "removeErasureCodingPolicy";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -2562,8 +2577,9 @@ public void removeErasureCodingPolicy(String ecPolicyName)
   @Override // ClientProtocol
   public void enableErasureCodingPolicy(String ecPolicyName)
       throws IOException {
+    String operationName = "enableErasureCodingPolicy";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -2580,8 +2596,9 @@ public void enableErasureCodingPolicy(String ecPolicyName)
   @Override // ClientProtocol
   public void disableErasureCodingPolicy(String ecPolicyName)
       throws IOException {
+    String operationName = "disableErasureCodingPolicy";
     checkNNStartup();
-    namesystem.checkSuperuserPrivilege();
+    namesystem.checkSuperuserPrivilege(operationName);
     final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 3ec7d61859143..a39648979c939 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -377,9 +377,10 @@ private void printDatanodeReplicaStatus(Block block,
    */
   public void fsck() throws AccessControlException {
     final long startTime = Time.monotonicNow();
+    String operationName = "fsck";
     try {
       if(blockIds != null) {
-        namenode.getNamesystem().checkSuperuserPrivilege();
+        namenode.getNamesystem().checkSuperuserPrivilege(operationName, path);
         StringBuilder sb = new StringBuilder();
         sb.append("FSCK started by " +
             UserGroupInformation.getCurrentUser() + " from " +
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
index 92e5ef1a0b86d..2d3adb2975324 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
@@ -65,8 +65,14 @@ static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr,
       boolean isRawPath)
       throws AccessControlException {
     final boolean isSuperUser = pc.isSuperUser();
+    final String xAttrString =
+        "XAttr [ns=" + xAttr.getNameSpace() + ", name=" + xAttr.getName() + "]";
     if (xAttr.getNameSpace() == XAttr.NameSpace.USER || 
         (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && isSuperUser)) {
+      if (isSuperUser) {
+        // call the external enforcer for audit.
+        pc.checkSuperuserPrivilege(xAttrString);
+      }
       return;
     }
     if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && isRawPath) {
@@ -75,14 +81,16 @@ static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr,
     if (XAttrHelper.getPrefixedName(xAttr).
         equals(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER)) {
       if (xAttr.getValue() != null) {
-        throw new AccessControlException("Attempt to set a value for '" +
+        // Notify external enforcer for audit
+        String errorMessage = "Attempt to set a value for '" +
             SECURITY_XATTR_UNREADABLE_BY_SUPERUSER +
-            "'. Values are not allowed for this xattr.");
+            "'. Values are not allowed for this xattr.";
+        pc.denyUserAccess(xAttrString, errorMessage);
       }
       return;
     }
-    throw new AccessControlException("User doesn't have permission for xattr: "
-        + XAttrHelper.getPrefixedName(xAttr));
+    pc.denyUserAccess(xAttrString, "User doesn't have permission for xattr: "
+            + XAttrHelper.getPrefixedName(xAttr));
   }
 
   static void checkPermissionForApi(FSPermissionChecker pc,

From 17be99f9f9c8987172a5a438379c998758ba48a8 Mon Sep 17 00:00:00 2001
From: lfengnan <lfengnan@uber.com>
Date: Thu, 15 Apr 2021 16:11:47 -0700
Subject: [PATCH 31/43] HDFS-15423 RBF: WebHDFS create shouldn't choose DN from
 all sub-clusters (#2903)

---
 .../router/RouterWebHdfsMethods.java          |  47 +++++-
 .../router/web/RouterWebHDFSContract.java     |   2 +
 .../federation/MiniRouterDFSCluster.java      |  19 +--
 .../router/TestRouterWebHdfsMethods.java      | 147 ++++++++++++++++++
 4 files changed, 198 insertions(+), 17 deletions(-)
 create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
index f6ac70c368a52..afc4a3d8fac21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
@@ -93,6 +93,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -104,6 +105,8 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * WebHDFS Router implementation. This is an extension of
@@ -453,21 +456,33 @@ private DatanodeInfo chooseDatanode(final Router router,
       final String path, final HttpOpParam.Op op, final long openOffset,
       final String excludeDatanodes) throws IOException {
     final RouterRpcServer rpcServer = getRPCServer(router);
-    DatanodeInfo[] dns = null;
+    DatanodeInfo[] dns = {};
+    String resolvedNs = "";
     try {
       dns = rpcServer.getCachedDatanodeReport(DatanodeReportType.LIVE);
     } catch (IOException e) {
       LOG.error("Cannot get the datanodes from the RPC server", e);
     }
 
+    if (op == PutOpParam.Op.CREATE) {
+      try {
+        resolvedNs = rpcServer.getCreateLocation(path).getNameserviceId();
+      } catch (IOException e) {
+        LOG.error("Cannot get the name service " +
+            "to create file for path {} ", path, e);
+      }
+    }
+
     HashSet<Node> excludes = new HashSet<Node>();
-    if (excludeDatanodes != null) {
-      Collection<String> collection =
-          getTrimmedStringCollection(excludeDatanodes);
-      for (DatanodeInfo dn : dns) {
-        if (collection.contains(dn.getName())) {
-          excludes.add(dn);
-        }
+    Collection<String> collection =
+        getTrimmedStringCollection(excludeDatanodes);
+    for (DatanodeInfo dn : dns) {
+      String ns = getNsFromDataNodeNetworkLocation(dn.getNetworkLocation());
+      if (collection.contains(dn.getName())) {
+        excludes.add(dn);
+      } else if (op == PutOpParam.Op.CREATE && !ns.equals(resolvedNs)) {
+        // for CREATE, the dest dn should be in the resolved ns
+        excludes.add(dn);
       }
     }
 
@@ -502,6 +517,22 @@ private DatanodeInfo chooseDatanode(final Router router,
     return getRandomDatanode(dns, excludes);
   }
 
+  /**
+   * Get the nameservice info from datanode network location.
+   * @param location network location with format `/ns0/rack1`
+   * @return nameservice this datanode is in
+   */
+  @VisibleForTesting
+  public static String getNsFromDataNodeNetworkLocation(String location) {
+    // network location should be in the format of /ns/rack
+    Pattern pattern = Pattern.compile("^/([^/]*)/");
+    Matcher matcher = pattern.matcher(location);
+    if (matcher.find()) {
+      return matcher.group(1);
+    }
+    return "";
+  }
+
   /**
    * Get a random Datanode from a subcluster.
    * @param dns Nodes to be chosen from.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
index 1d308073290d4..6b90faecc78f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java
@@ -64,6 +64,8 @@ public static void createCluster(Configuration conf) throws IOException {
       conf.addResource(CONTRACT_WEBHDFS_XML);
 
       cluster = new MiniRouterDFSCluster(true, 2, conf);
+      cluster.setIndependentDNs();
+      cluster.setNumDatanodesPerNameservice(3);
 
       // Start NNs and DNs and wait until ready
       cluster.startCluster(conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
index 896d08f2c49b6..8a7a03e018b95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
@@ -774,6 +774,15 @@ public void startCluster(Configuration overrideConf) {
       }
       topology.setFederation(true);
 
+      // Generate conf for namenodes and datanodes
+      String ns0 = nameservices.get(0);
+      Configuration nnConf = generateNamenodeConfiguration(ns0);
+      if (overrideConf != null) {
+        nnConf.addResource(overrideConf);
+        // Router also uses this configurations as initial values.
+        routerConf = new Configuration(overrideConf);
+      }
+
       // Set independent DNs across subclusters
       int numDNs = nameservices.size() * numDatanodesPerNameservice;
       Configuration[] dnConfs = null;
@@ -781,7 +790,7 @@ public void startCluster(Configuration overrideConf) {
         dnConfs = new Configuration[numDNs];
         int dnId = 0;
         for (String nsId : nameservices) {
-          Configuration subclusterConf = new Configuration();
+          Configuration subclusterConf = new Configuration(nnConf);
           subclusterConf.set(DFS_INTERNAL_NAMESERVICES_KEY, nsId);
           for (int i = 0; i < numDatanodesPerNameservice; i++) {
             dnConfs[dnId] = subclusterConf;
@@ -791,14 +800,6 @@ public void startCluster(Configuration overrideConf) {
       }
 
       // Start mini DFS cluster
-      String ns0 = nameservices.get(0);
-      Configuration nnConf = generateNamenodeConfiguration(ns0);
-      if (overrideConf != null) {
-        nnConf.addResource(overrideConf);
-        // Router also uses this configurations as initial values.
-        routerConf = new Configuration(overrideConf);
-      }
-
       cluster = new MiniDFSCluster.Builder(nnConf)
           .numDataNodes(numDNs)
           .nnTopology(topology)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
new file mode 100644
index 0000000000000..7028928041452
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createMountTableEntry;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.FileNotFoundException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Collections;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test suite for Router Web Hdfs methods.
+ */
+public class TestRouterWebHdfsMethods {
+  static final Logger LOG =
+      LoggerFactory.getLogger(TestRouterWebHdfsMethods.class);
+
+  private static StateStoreDFSCluster cluster;
+  private static RouterContext router;
+  private static String httpUri;
+
+  @BeforeClass
+  public static void globalSetUp() throws Exception {
+    cluster = new StateStoreDFSCluster(false, 2);
+    Configuration conf = new RouterConfigBuilder()
+        .stateStore()
+        .rpc()
+        .http()
+        .admin()
+        .build();
+    cluster.addRouterOverrides(conf);
+    cluster.startCluster();
+    cluster.startRouters();
+    cluster.waitClusterUp();
+    router = cluster.getRandomRouter();
+    httpUri = "http://"+router.getHttpAddress();
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @Test
+  public void testWebHdfsCreate() throws Exception {
+    // the file is created at default ns (ns0)
+    String path = "/tmp/file";
+    URL url = new URL(getUri(path));
+    LOG.info("URL: {}", url);
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
+    verifyFile("ns0", path, true);
+    verifyFile("ns1", path, false);
+    conn.disconnect();
+  }
+
+  @Test
+  public void testWebHdfsCreateWithMounts() throws Exception {
+    // the file is created at mounted ns (ns1)
+    String mountPoint = "/tmp-ns1";
+    String path = "/tmp-ns1/file";
+    createMountTableEntry(
+        router.getRouter(), mountPoint,
+        DestinationOrder.RANDOM, Collections.singletonList("ns1"));
+    URL url = new URL(getUri(path));
+    LOG.info("URL: {}", url);
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
+    verifyFile("ns1", path, true);
+    verifyFile("ns0", path, false);
+    conn.disconnect();
+  }
+
+  private String getUri(String path) {
+    final String user = System.getProperty("user.name");
+    final StringBuilder uri = new StringBuilder(httpUri);
+    uri.append("/webhdfs/v1").
+        append(path).
+        append("?op=CREATE").
+        append("&user.name=" + user);
+    return uri.toString();
+  }
+
+  private void verifyFile(String ns, String path, boolean shouldExist)
+      throws Exception {
+    FileSystem fs = cluster.getNamenode(ns, null).getFileSystem();
+    try {
+      fs.getFileStatus(new Path(path));
+      if (!shouldExist) {
+        fail(path + " should not exist in ns " + ns);
+      }
+    } catch (FileNotFoundException e) {
+      if (shouldExist) {
+        fail(path + " should exist in ns " + ns);
+      }
+    }
+  }
+
+  @Test
+  public void testGetNsFromDataNodeNetworkLocation() {
+    assertEquals("ns0", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("/ns0/rack-info1"));
+    assertEquals("ns0", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("/ns0/row1/rack-info1"));
+    assertEquals("", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("/row0"));
+    assertEquals("", RouterWebHdfsMethods
+        .getNsFromDataNodeNetworkLocation("whatever-rack-info1"));
+  }
+}

From f0241ec2161f6eccdb9bdaf1cbcbee55be379217 Mon Sep 17 00:00:00 2001
From: Akira Ajisaka <aajisaka@apache.org>
Date: Fri, 16 Apr 2021 13:26:45 +0900
Subject: [PATCH 32/43] HDFS-15977. Call explicit_bzero only if it is
 available. (#2914)

Reviewed-by: Masatake Iwasaki <iwasakims@apache.org>
Reviewed-by: Inigo Goiri <inigoiri@apache.org>
---
 .../src/main/native/libhdfspp/CMakeLists.txt                | 6 ++++++
 .../main/native/libhdfspp/lib/x-platform/syscall_linux.cc   | 5 +++++
 2 files changed, 11 insertions(+)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index b03ba68172228..d23f84941db9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -51,6 +51,7 @@ find_package(GSasl)
 find_package(Threads)
 
 include(CheckCXXSourceCompiles)
+include(CheckSymbolExists)
 
 # Download and build gtest
 configure_file(CMakeLists-gtest.txt.in googletest-download/CMakeLists.txt)
@@ -168,6 +169,11 @@ else (NOT NO_SASL)
     message(STATUS "Compiling with NO SASL SUPPORT")
 endif (NOT NO_SASL)
 
+check_symbol_exists(explicit_bzero "string.h" HAVE_EXPLICIT_BZERO)
+if(HAVE_EXPLICIT_BZERO)
+    add_definitions(-DHAVE_EXPLICIT_BZERO)
+endif()
+
 add_definitions(-DASIO_STANDALONE -DASIO_CPP11_DATE_TIME)
 
 # Disable optimizations if compiling debug
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index 9903bbe853ba3..ff02d2fa65cf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -47,7 +47,12 @@ bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
 void XPlatform::Syscall::ClearBufferSafely(void* buffer,
                                            const size_t sz_bytes) {
   if (buffer != nullptr) {
+#ifdef HAVE_EXPLICIT_BZERO
     explicit_bzero(buffer, sz_bytes);
+#else
+    // fallback to bzero
+    bzero(buffer, sz_bytes);
+#endif
   }
 }
 

From f1827986951a975a2ff13ca1a6a032cbd50b95c0 Mon Sep 17 00:00:00 2001
From: Viraj Jasani <vjasani@apache.org>
Date: Fri, 16 Apr 2021 12:36:01 +0530
Subject: [PATCH 33/43] HADOOP-17633. Bump json-smart to 2.4.2 and
 nimbus-jose-jwt to 9.8 due to CVEs (#2895). Contributed by Viraj Jasani.

Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
---
 LICENSE-binary         | 4 ++--
 hadoop-project/pom.xml | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/LICENSE-binary b/LICENSE-binary
index 198f97c584393..5af4b60f6cf01 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -240,7 +240,7 @@ com.google.guava:guava:20.0
 com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
-com.nimbusds:nimbus-jose-jwt:4.41.1
+com.nimbusds:nimbus-jose-jwt:9.8.1
 com.squareup.okhttp:okhttp:2.7.5
 com.squareup.okio:okio:1.6.0
 com.zaxxer:HikariCP-java7:2.4.12
@@ -283,7 +283,7 @@ javax.inject:javax.inject:1
 log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
-net.minidev:json-smart:2.3
+net.minidev:json-smart:2.4.2
 org.apache.avro:avro:1.7.7
 org.apache.commons:commons-collections4:4.2
 org.apache.commons:commons-compress:1.19
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 5f81d75605a3d..f09bf533beaab 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -210,6 +210,8 @@
     <openssl-wildfly.version>1.0.7.Final</openssl-wildfly.version>
     <jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
     <woodstox.version>5.3.0</woodstox.version>
+    <json-smart.version>2.4.2</json-smart.version>
+    <nimbus-jose-jwt.version>9.8.1</nimbus-jose-jwt.version>
   </properties>
 
   <dependencyManagement>
@@ -1566,7 +1568,7 @@
       <dependency>
           <groupId>com.nimbusds</groupId>
           <artifactId>nimbus-jose-jwt</artifactId>
-          <version>7.9</version>
+          <version>${nimbus-jose-jwt.version}</version>
           <scope>compile</scope>
           <exclusions>
           <exclusion>
@@ -1589,7 +1591,7 @@
           -->
         <groupId>net.minidev</groupId>
         <artifactId>json-smart</artifactId>
-        <version>2.3</version>
+        <version>${json-smart.version}</version>
       </dependency>
       <dependency>
         <groupId>org.skyscreamer</groupId>

From 14816be0b1bce4a472e38ed9d15acf294eaa5356 Mon Sep 17 00:00:00 2001
From: zhangshuyan0 <81411509+zhangshuyan0@users.noreply.github.com>
Date: Sat, 17 Apr 2021 00:08:31 +0800
Subject: [PATCH 34/43] HDFS-15963. Unreleased volume references cause an
 infinite loop. (#2889) Contributed by Shuyan Zhang.

Reviewed-by: Wei-Chiu Chuang <weichiu@apache.org>
Reviewed-by: He Xiaoqiao <hexiaoqiao@apache.org>
---
 .../hdfs/server/datanode/BlockSender.java     |  1 +
 .../impl/FsDatasetAsyncDiskService.java       | 71 +++++++++++--------
 .../datanode/fsdataset/impl/FsVolumeImpl.java |  2 +-
 .../impl/RamDiskAsyncLazyPersistService.java  | 29 +++++---
 .../hadoop/hdfs/TestDataTransferProtocol.java | 55 ++++++++++++++
 .../fsdataset/impl/TestFsDatasetImpl.java     | 34 +++++++++
 .../fsdataset/impl/TestLazyPersistFiles.java  | 37 ++++++++++
 7 files changed, 188 insertions(+), 41 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index fb6d83fcc6bd6..bb75e3aceb6be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -432,6 +432,7 @@ class BlockSender implements java.io.Closeable {
       ris = new ReplicaInputStreams(
           blockIn, checksumIn, volumeRef, fileIoProvider);
     } catch (IOException ioe) {
+      IOUtils.cleanupWithLogger(null, volumeRef);
       IOUtils.closeStream(this);
       org.apache.commons.io.IOUtils.closeQuietly(blockIn);
       org.apache.commons.io.IOUtils.closeQuietly(checksumIn);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 2a89a80d17a25..706c078e648b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -167,18 +167,26 @@ synchronized long countPendingDeletions() {
    * Execute the task sometime in the future, using ThreadPools.
    */
   synchronized void execute(FsVolumeImpl volume, Runnable task) {
-    if (executors == null) {
-      throw new RuntimeException("AsyncDiskService is already shutdown");
-    }
-    if (volume == null) {
-      throw new RuntimeException("A null volume does not have a executor");
-    }
-    ThreadPoolExecutor executor = executors.get(volume.getStorageID());
-    if (executor == null) {
-      throw new RuntimeException("Cannot find volume " + volume
-          + " for execution of task " + task);
-    } else {
-      executor.execute(task);
+    try {
+      if (executors == null) {
+        throw new RuntimeException("AsyncDiskService is already shutdown");
+      }
+      if (volume == null) {
+        throw new RuntimeException("A null volume does not have a executor");
+      }
+      ThreadPoolExecutor executor = executors.get(volume.getStorageID());
+      if (executor == null) {
+        throw new RuntimeException("Cannot find volume " + volume
+            + " for execution of task " + task);
+      } else {
+        executor.execute(task);
+      }
+    } catch (RuntimeException re) {
+      if (task instanceof ReplicaFileDeleteTask) {
+        IOUtils.cleanupWithLogger(null,
+            ((ReplicaFileDeleteTask) task).volumeRef);
+      }
+      throw re;
     }
   }
   
@@ -314,28 +322,31 @@ private boolean moveFiles() {
 
     @Override
     public void run() {
-      final long blockLength = replicaToDelete.getBlockDataLength();
-      final long metaLength = replicaToDelete.getMetadataLength();
-      boolean result;
+      try {
+        final long blockLength = replicaToDelete.getBlockDataLength();
+        final long metaLength = replicaToDelete.getMetadataLength();
+        boolean result;
 
-      result = (trashDirectory == null) ? deleteFiles() : moveFiles();
+        result = (trashDirectory == null) ? deleteFiles() : moveFiles();
 
-      if (!result) {
-        LOG.warn("Unexpected error trying to "
-            + (trashDirectory == null ? "delete" : "move")
-            + " block " + block.getBlockPoolId() + " " + block.getLocalBlock()
-            + " at file " + replicaToDelete.getBlockURI() + ". Ignored.");
-      } else {
-        if(block.getLocalBlock().getNumBytes() != BlockCommand.NO_ACK){
-          datanode.notifyNamenodeDeletedBlock(block, volume.getStorageID());
+        if (!result) {
+          LOG.warn("Unexpected error trying to "
+              + (trashDirectory == null ? "delete" : "move")
+              + " block " + block.getBlockPoolId() + " " + block.getLocalBlock()
+              + " at file " + replicaToDelete.getBlockURI() + ". Ignored.");
+        } else {
+          if (block.getLocalBlock().getNumBytes() != BlockCommand.NO_ACK) {
+            datanode.notifyNamenodeDeletedBlock(block, volume.getStorageID());
+          }
+          volume.onBlockFileDeletion(block.getBlockPoolId(), blockLength);
+          volume.onMetaFileDeletion(block.getBlockPoolId(), metaLength);
+          LOG.info("Deleted " + block.getBlockPoolId() + " " +
+              block.getLocalBlock() + " URI " + replicaToDelete.getBlockURI());
         }
-        volume.onBlockFileDeletion(block.getBlockPoolId(), blockLength);
-        volume.onMetaFileDeletion(block.getBlockPoolId(), metaLength);
-        LOG.info("Deleted " + block.getBlockPoolId() + " "
-            + block.getLocalBlock() + " URI " + replicaToDelete.getBlockURI());
+        updateDeletedBlockId(block);
+      } finally {
+        IOUtils.cleanupWithLogger(null, this.volumeRef);
       }
-      updateDeletedBlockId(block);
-      IOUtils.cleanupWithLogger(null, volumeRef);
     }
   }
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 07e14fb04e48e..6681f6fd64c0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -319,7 +319,7 @@ private void checkReference() {
   }
 
   @VisibleForTesting
-  int getReferenceCount() {
+  public int getReferenceCount() {
     return this.reference.getReferenceCount();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
index a77faf2cec8bc..0d42ae99e358e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
+import org.apache.hadoop.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -153,16 +154,24 @@ synchronized boolean queryVolume(FsVolumeImpl volume) {
    * Execute the task sometime in the future, using ThreadPools.
    */
   synchronized void execute(String storageId, Runnable task) {
-    if (executors == null) {
-      throw new RuntimeException(
-          "AsyncLazyPersistService is already shutdown");
-    }
-    ThreadPoolExecutor executor = executors.get(storageId);
-    if (executor == null) {
-      throw new RuntimeException("Cannot find root storage volume with id " +
-          storageId + " for execution of task " + task);
-    } else {
-      executor.execute(task);
+    try {
+      if (executors == null) {
+        throw new RuntimeException(
+            "AsyncLazyPersistService is already shutdown");
+      }
+      ThreadPoolExecutor executor = executors.get(storageId);
+      if (executor == null) {
+        throw new RuntimeException("Cannot find root storage volume with id " +
+            storageId + " for execution of task " + task);
+      } else {
+        executor.execute(task);
+      }
+    } catch (RuntimeException re) {
+      if (task instanceof ReplicaLazyPersistTask) {
+        IOUtils.cleanupWithLogger(null,
+            ((ReplicaLazyPersistTask) task).targetVolume);
+      }
+      throw re;
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
index b9da5f446f92f..b1a675c77b62a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
@@ -33,6 +33,9 @@
 import java.nio.ByteBuffer;
 import java.util.Random;
 
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -562,4 +565,56 @@ void writeBlock(ExtendedBlock block, BlockConstructionStage stage,
         checksum, CachingStrategy.newDefaultStrategy(), false, false,
         null, null, new String[0]);
   }
+
+  @Test(timeout = 30000)
+  public void testReleaseVolumeRefIfExceptionThrown()
+      throws IOException, InterruptedException {
+    Path file = new Path("dataprotocol.dat");
+    int numDataNodes = 1;
+
+    Configuration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, numDataNodes);
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
+        numDataNodes).build();
+    try {
+      cluster.waitActive();
+      datanode = cluster.getFileSystem().getDataNodeStats(
+          DatanodeReportType.LIVE)[0];
+      dnAddr = NetUtils.createSocketAddr(datanode.getXferAddr());
+      FileSystem fileSys = cluster.getFileSystem();
+
+      int fileLen = Math.min(
+          conf.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096), 4096);
+
+      DFSTestUtil.createFile(fileSys, file, fileLen, fileLen,
+          fileSys.getDefaultBlockSize(file),
+          fileSys.getDefaultReplication(file), 0L);
+
+      // Get the first blockid for the file.
+      final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fileSys, file);
+
+      String bpid = cluster.getNamesystem().getBlockPoolId();
+      ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.getLocalBlock());
+      sendBuf.reset();
+      recvBuf.reset();
+
+      // Delete the meta file to create a exception in BlockSender constructor.
+      DataNode dn = cluster.getDataNodes().get(0);
+      cluster.getMaterializedReplica(0, blk).deleteMeta();
+
+      FsVolumeImpl volume = (FsVolumeImpl) DataNodeTestUtils.getFSDataset(
+          dn).getVolume(blk);
+      int beforeCnt = volume.getReferenceCount();
+
+      sender.copyBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN);
+      sendRecvData("Copy a block.", false);
+      Thread.sleep(3000);
+
+      int afterCnt = volume.getReferenceCount();
+      assertEquals(beforeCnt, afterCnt);
+
+    } finally {
+      cluster.shutdown();
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 6ae6248d3f997..778ef97180b41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.fs.DF;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner;
+import org.apache.hadoop.hdfs.server.datanode.LocalReplica;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
 
 import java.io.OutputStream;
@@ -1805,4 +1806,37 @@ public void testNotifyNamenodeMissingOrNewBlock() throws Exception {
       cluster.shutdown();
     }
   }
+
+  @Test(timeout = 20000)
+  public void testReleaseVolumeRefIfExceptionThrown() throws IOException {
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(
+        new HdfsConfiguration()).build();
+    cluster.waitActive();
+    FsVolumeImpl vol = (FsVolumeImpl) dataset.getFsVolumeReferences().get(0);
+    ExtendedBlock eb;
+    ReplicaInfo info;
+    int beforeCnt = 0;
+    try {
+      List<Block> blockList = new ArrayList<Block>();
+      eb = new ExtendedBlock(BLOCKPOOL, 1, 1, 1001);
+      info = new FinalizedReplica(
+          eb.getLocalBlock(), vol, vol.getCurrentDir().getParentFile());
+      dataset.volumeMap.add(BLOCKPOOL, info);
+      ((LocalReplica) info).getBlockFile().createNewFile();
+      ((LocalReplica) info).getMetaFile().createNewFile();
+      blockList.add(info);
+
+      // Create a runtime exception.
+      dataset.asyncDiskService.shutdown();
+
+      beforeCnt = vol.getReferenceCount();
+      dataset.invalidate(BLOCKPOOL, blockList.toArray(new Block[0]));
+
+    } catch (RuntimeException re) {
+      int afterCnt = vol.getReferenceCount();
+      assertEquals(beforeCnt, afterCnt);
+    } finally {
+      cluster.shutdown();
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index c0b4b17ea9566..14ed26e9b5544 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -18,6 +18,9 @@
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.ThreadUtil;
 import org.junit.Assert;
@@ -280,4 +283,38 @@ public void run() {
       }
     }
   }
+
+  @Test(timeout = 20000)
+  public void testReleaseVolumeRefIfExceptionThrown()
+      throws IOException, InterruptedException {
+    getClusterBuilder().setRamDiskReplicaCapacity(2).build();
+    final String methodName = GenericTestUtils.getMethodName();
+    final int seed = 0xFADED;
+    Path path = new Path("/" + methodName + ".Writer.File.dat");
+
+    DataNode dn = cluster.getDataNodes().get(0);
+    FsDatasetSpi.FsVolumeReferences volumes =
+        DataNodeTestUtils.getFSDataset(dn).getFsVolumeReferences();
+    int[] beforeCnts = new int[volumes.size()];
+    FsDatasetImpl ds = (FsDatasetImpl) DataNodeTestUtils.getFSDataset(dn);
+
+    // Create a runtime exception.
+    ds.asyncLazyPersistService.shutdown();
+    for (int i = 0; i < volumes.size(); ++i) {
+      beforeCnts[i] = ((FsVolumeImpl) volumes.get(i)).getReferenceCount();
+    }
+
+    makeRandomTestFile(path, BLOCK_SIZE, true, seed);
+    Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+    for (int i = 0; i < volumes.size(); ++i) {
+      int afterCnt = ((FsVolumeImpl) volumes.get(i)).getReferenceCount();
+      // LazyWriter keeps trying to save copies even if
+      // asyncLazyPersistService is already shutdown.
+      // If we do not release references, the number of
+      // references will increase infinitely.
+      Assert.assertTrue(
+          beforeCnts[i] == afterCnt || beforeCnts[i] == (afterCnt - 1));
+    }
+  }
 }

From aed13f0f42fefe30a53eb73c65c2072a031f173e Mon Sep 17 00:00:00 2001
From: Eric Badger <ebadger@verizonmedia.com>
Date: Fri, 16 Apr 2021 17:15:33 +0000
Subject: [PATCH 35/43] Revert "HDFS-15971. Make mkstemp cross platform
 (#2898)"

This reverts commit b088f46b684ccefb401b9a3be197d75e3bc64f6b.
---
 .../src/CMakeLists.txt                        |  3 +-
 .../libhdfs-tests/test_libhdfs_mini_stress.c  |  4 +-
 .../libhdfspp/lib/x-platform/c-api/syscall.cc | 22 +------
 .../libhdfspp/lib/x-platform/c-api/syscall.h  |  3 -
 .../native/libhdfspp/lib/x-platform/syscall.h | 29 ---------
 .../libhdfspp/lib/x-platform/syscall_linux.cc | 11 ----
 .../lib/x-platform/syscall_windows.cc         | 30 ---------
 .../native/libhdfspp/tests/CMakeLists.txt     | 13 ++--
 .../libhdfspp/tests/configuration_test.cc     | 23 +++----
 .../libhdfspp/tests/configuration_test.h      | 64 ++++---------------
 .../libhdfspp/tests/hdfs_builder_test.cc      |  4 +-
 .../tests/hdfs_configuration_test.cc          | 16 ++---
 .../tests/x-platform/syscall_common_test.cc   | 18 ------
 13 files changed, 41 insertions(+), 199 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index d8c2012fb88de..7c076bd790085 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -91,8 +91,7 @@ function(build_libhdfs_test NAME LIBRARY)
             list(APPEND FILES ${CMAKE_SOURCE_DIR}/main/native/libhdfs-tests/${FIL})
         endif()
     endforeach()
-    add_executable("${NAME}_${LIBRARY}" $<TARGET_OBJECTS:x_platform_obj_c_api> $<TARGET_OBJECTS:x_platform_obj> ${FILES})
-    target_include_directories("${NAME}_${LIBRARY}" PRIVATE main/native/libhdfspp/lib)
+    add_executable("${NAME}_${LIBRARY}" ${FILES})
 endfunction()
 
 function(add_libhdfs_test NAME LIBRARY)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
index 1641470733f2c..846852bfd0e88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
@@ -22,7 +22,6 @@
 #include "hdfspp/hdfs_ext.h"
 #include "native_mini_dfs.h"
 #include "os/thread.h"
-#include "x-platform/c-api/syscall.h"
 
 #include <errno.h>
 #include <inttypes.h>
@@ -127,8 +126,7 @@ static int hdfsCurlData(const char *host, const tPort port, const char *dirNm,
   EXPECT_NONNULL(pw = getpwuid(uid));
 
   int fd = -1;
-  EXPECT_NONNEGATIVE(fd = x_platform_syscall_create_and_open_temp_file(
-                         tmpFile, sizeof tmpFile));
+  EXPECT_NONNEGATIVE(fd = mkstemp(tmpFile));
 
   tSize sz = 0;
   while (sz < fileSz) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc
index 0bb5fc15ef631..cca9f6a024807 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.cc
@@ -18,26 +18,6 @@
 
 #include "x-platform/syscall.h"
 
-#include <algorithm>
-#include <vector>
-
-extern "C" {
-int x_platform_syscall_write_to_stdout(const char* msg) {
+extern "C" int x_platform_syscall_write_to_stdout(const char* msg) {
   return XPlatform::Syscall::WriteToStdout(msg) ? 1 : 0;
 }
-
-int x_platform_syscall_create_and_open_temp_file(char* pattern,
-                                                 const size_t pattern_len) {
-  std::vector<char> pattern_vec(pattern, pattern + pattern_len);
-
-  const auto fd = XPlatform::Syscall::CreateAndOpenTempFile(pattern_vec);
-  if (fd != -1) {
-    std::copy_n(pattern_vec.begin(), pattern_len, pattern);
-  }
-  return fd;
-}
-
-int x_platform_syscall_close_file(const int fd) {
-  return XPlatform::Syscall::CloseFile(fd);
-}
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h
index 93878b144db98..be905ae1d364e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h
@@ -24,8 +24,5 @@
  */
 
 int x_platform_syscall_write_to_stdout(const char* msg);
-int x_platform_syscall_create_and_open_temp_file(char* pattern,
-                                                 size_t pattern_len);
-int x_platform_syscall_close_file(int fd);
 
 #endif  // NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_SYSCALL_H
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
index 9959f215b2068..3a77f2dd6d026 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
@@ -20,7 +20,6 @@
 #define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_SYSCALL
 
 #include <string>
-#include <vector>
 
 /**
  * The {@link XPlatform} namespace contains components that
@@ -85,34 +84,6 @@ class Syscall {
   static bool StringCompareIgnoreCase(const std::string& a,
                                       const std::string& b);
 
-  /**
-   * Creates and opens a temporary file with a given {@link pattern}.
-   * The {@link pattern} must end with a minimum of 6 'X' characters.
-   * This function will first modify the last 6 'X' characters with
-   * random character values, which serve as the temporary file name.
-   * Subsequently opens the file and returns the file descriptor for
-   * the same. The behaviour of this function is the same as that of
-   * POSIX mkstemp function. The file must be later closed by the
-   * application and is not handled by this function.
-   *
-   * @param pattern the pattern to be used for the temporary filename.
-   * @returns an integer representing the file descriptor for the
-   * opened temporary file. Returns -1 in the case of error and sets
-   * the global errno with the appropriate error code.
-   */
-  static int CreateAndOpenTempFile(std::vector<char>& pattern);
-
-  /**
-   * Closes the file corresponding to given {@link file_descriptor}.
-   *
-   * @param file_descriptor the file descriptor of the file to close.
-   * @returns a boolean indicating the status of the call to this
-   * function. true if it's a success, false in the case of an error.
-   * The global errno is set if the call to this function was not
-   * successful.
-   */
-  static bool CloseFile(int file_descriptor);
-
  private:
   static bool WriteToStdoutImpl(const char* message);
 };
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index ff02d2fa65cf5..59d93c40b7681 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -21,7 +21,6 @@
 #include <unistd.h>
 
 #include <cstring>
-#include <vector>
 
 #include "syscall.h"
 
@@ -60,13 +59,3 @@ bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
                                                  const std::string& b) {
   return strcasecmp(a.c_str(), b.c_str()) == 0;
 }
-
-int XPlatform::Syscall::CreateAndOpenTempFile(std::vector<char>& pattern) {
-  // Make space for mkstemp to add NULL character at the end
-  pattern.resize(pattern.size() + 1);
-  return mkstemp(pattern.data());
-}
-
-bool XPlatform::Syscall::CloseFile(const int file_descriptor) {
-  return close(file_descriptor) == 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
index b5ddd04373841..2cd9e9d5157df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
@@ -19,14 +19,7 @@
 #include <Shlwapi.h>
 #include <WinBase.h>
 #include <Windows.h>
-#include <fcntl.h>
-#include <io.h>
-#include <share.h>
-#include <sys/stat.h>
-#include <sys/types.h>
 
-#include <cerrno>
-#include <cstdlib>
 #include <cstring>
 
 #include "syscall.h"
@@ -71,26 +64,3 @@ bool XPlatform::Syscall::StringCompareIgnoreCase(const std::string& a,
                                                  const std::string& b) {
   return _stricmp(a.c_str(), b.c_str()) == 0;
 }
-
-int XPlatform::Syscall::CreateAndOpenTempFile(std::vector<char>& pattern) {
-  if (_set_errno(0) != 0) {
-    return -1;
-  }
-
-  // Make space for _mktemp_s to add NULL character at the end
-  pattern.resize(pattern.size() + 1);
-  if (_mktemp_s(pattern.data(), pattern.size()) != 0) {
-    return -1;
-  }
-
-  auto fd{-1};
-  if (_sopen_s(&fd, pattern.data(), _O_RDWR | _O_CREAT | _O_EXCL, _SH_DENYNO,
-               _S_IREAD | _S_IWRITE) != 0) {
-    return -1;
-  }
-  return fd;
-}
-
-bool XPlatform::Syscall::CloseFile(const int file_descriptor) {
-  return _close(file_descriptor) == 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index d37fcc29f9cbd..32f75f474b8d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -96,13 +96,11 @@ add_executable(node_exclusion_test node_exclusion_test.cc)
 target_link_libraries(node_exclusion_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(node_exclusion node_exclusion_test)
 
-add_executable(configuration_test $<TARGET_OBJECTS:x_platform_obj> configuration_test.cc)
-target_include_directories(configuration_test PRIVATE ../lib)
+add_executable(configuration_test configuration_test.cc)
 target_link_libraries(configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(configuration configuration_test)
 
-add_executable(hdfs_configuration_test $<TARGET_OBJECTS:x_platform_obj> hdfs_configuration_test.cc)
-target_include_directories(hdfs_configuration_test PRIVATE ../lib)
+add_executable(hdfs_configuration_test hdfs_configuration_test.cc)
 target_link_libraries(hdfs_configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfs_configuration hdfs_configuration_test)
 
@@ -110,13 +108,11 @@ add_executable(hdfspp_errors_test hdfspp_errors.cc)
 target_link_libraries(hdfspp_errors_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfspp_errors hdfspp_errors_test)
 
-add_executable(hdfs_builder_test $<TARGET_OBJECTS:x_platform_obj> hdfs_builder_test.cc)
-target_include_directories(hdfs_builder_test PRIVATE ../lib)
+add_executable(hdfs_builder_test hdfs_builder_test.cc)
 target_link_libraries(hdfs_builder_test test_common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfs_builder_test hdfs_builder_test)
 
 add_executable(logging_test logging_test.cc $<TARGET_OBJECTS:x_platform_obj>)
-target_include_directories(logging_test PRIVATE ../lib)
 target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(logging_test logging_test)
 
@@ -128,8 +124,7 @@ add_executable(user_lock_test user_lock_test.cc)
 target_link_libraries(user_lock_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(user_lock user_lock_test)
 
-add_executable(hdfs_config_connect_bugs_test $<TARGET_OBJECTS:x_platform_obj> hdfs_config_connect_bugs.cc)
-target_include_directories(hdfs_config_connect_bugs_test PRIVATE ../lib)
+add_executable(hdfs_config_connect_bugs_test hdfs_config_connect_bugs.cc)
 target_link_libraries(hdfs_config_connect_bugs_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfs_config_connect_bugs hdfs_config_connect_bugs_test)
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
index 3bf2524354b6b..9534204c92ca0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.cc
@@ -299,12 +299,11 @@ TEST(ConfigurationTest, TestFileReads)
   // Single stream
   {
     TempFile tempFile;
-    writeSimpleConfig(tempFile.GetFileName(), "key1", "value1");
+    writeSimpleConfig(tempFile.filename, "key1", "value1");
 
     ConfigurationLoader config_loader;
     config_loader.ClearSearchPath();
-    optional<Configuration> config =
-        config_loader.LoadFromFile<Configuration>(tempFile.GetFileName());
+    optional<Configuration> config = config_loader.LoadFromFile<Configuration>(tempFile.filename);
     EXPECT_TRUE(config && "Parse first stream");
     EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
   }
@@ -312,18 +311,16 @@ TEST(ConfigurationTest, TestFileReads)
   // Multiple files
   {
     TempFile tempFile;
-    writeSimpleConfig(tempFile.GetFileName(), "key1", "value1");
+    writeSimpleConfig(tempFile.filename, "key1", "value1");
 
     ConfigurationLoader loader;
-    optional<Configuration> config =
-        loader.LoadFromFile<Configuration>(tempFile.GetFileName());
+    optional<Configuration> config = loader.LoadFromFile<Configuration>(tempFile.filename);
     ASSERT_TRUE(config && "Parse first stream");
     EXPECT_EQ("value1", config->GetWithDefault("key1", ""));
 
     TempFile tempFile2;
-    writeSimpleConfig(tempFile2.GetFileName(), "key2", "value2");
-    optional<Configuration> config2 =
-        loader.OverlayResourceFile(*config, tempFile2.GetFileName());
+    writeSimpleConfig(tempFile2.filename, "key2", "value2");
+    optional<Configuration> config2 = loader.OverlayResourceFile(*config, tempFile2.filename);
     ASSERT_TRUE(config2 && "Parse second stream");
     EXPECT_EQ("value1", config2->GetWithDefault("key1", ""));
     EXPECT_EQ("value2", config2->GetWithDefault("key2", ""));
@@ -353,13 +350,13 @@ TEST(ConfigurationTest, TestFileReads)
   {
     TempDir tempDir1;
     TempFile tempFile1(tempDir1.path + "/file1.xml");
-    writeSimpleConfig(tempFile1.GetFileName(), "key1", "value1");
+    writeSimpleConfig(tempFile1.filename, "key1", "value1");
     TempDir tempDir2;
     TempFile tempFile2(tempDir2.path + "/file2.xml");
-    writeSimpleConfig(tempFile2.GetFileName(), "key2", "value2");
+    writeSimpleConfig(tempFile2.filename, "key2", "value2");
     TempDir tempDir3;
     TempFile tempFile3(tempDir3.path + "/file3.xml");
-    writeSimpleConfig(tempFile3.GetFileName(), "key3", "value3");
+    writeSimpleConfig(tempFile3.filename, "key3", "value3");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir1.path + ":" + tempDir2.path + ":" + tempDir3.path);
@@ -380,7 +377,7 @@ TEST(ConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
+    writeSimpleConfig(coreSite.filename, "key1", "value1");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
index 23fc0d31047b3..9ad11b70cc8dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/configuration_test.h
@@ -21,19 +21,11 @@
 #include "hdfspp/config_parser.h"
 #include "common/configuration.h"
 #include "common/configuration_loader.h"
-#include "x-platform/syscall.h"
-
 #include <cstdio>
 #include <fstream>
 #include <istream>
-#include <string>
-#include <utility>
-#include <vector>
-
 #include <ftw.h>
-#include <unistd.h>
 #include <gmock/gmock.h>
-#include <gtest/gtest.h>
 
 namespace hdfs {
 
@@ -115,52 +107,24 @@ void writeDamagedConfig(const std::string& filename, Args... args) {
 
 // TempDir: is deleted on destruction
 class TempFile {
- public:
-  TempFile() {
-    std::vector<char> tmp_buf(filename_.begin(), filename_.end());
-    fd_ = XPlatform::Syscall::CreateAndOpenTempFile(tmp_buf);
-    EXPECT_NE(fd_, -1);
-    filename_.assign(tmp_buf.data());
-  }
-
-  TempFile(std::string fn) : filename_(std::move(fn)) {}
-
-  TempFile(const TempFile& other) = default;
-
-  TempFile(TempFile&& other) noexcept
-      : filename_{std::move(other.filename_)}, fd_{other.fd_} {}
-
-  TempFile& operator=(const TempFile& other) {
-    if (&other != this) {
-      filename_ = other.filename_;
-      fd_ = other.fd_;
-    }
-    return *this;
-  }
-
-  TempFile& operator=(TempFile&& other) noexcept {
-    if (&other != this) {
-      filename_ = std::move(other.filename_);
-      fd_ = other.fd_;
-    }
-    return *this;
+public:
+  std::string filename;
+  char        fn_buffer[128];
+  int         tempFileHandle;
+  TempFile() : tempFileHandle(-1) {
+    strncpy(fn_buffer, "/tmp/test_XXXXXXXXXX", sizeof(fn_buffer));
+    tempFileHandle = mkstemp(fn_buffer);
+    EXPECT_NE(-1, tempFileHandle);
+    filename = fn_buffer;
   }
-
-  [[nodiscard]] const std::string& GetFileName() const { return filename_; }
-
-  ~TempFile() {
-    if (-1 != fd_) {
-      EXPECT_NE(XPlatform::Syscall::CloseFile(fd_), -1);
-    }
-
-    unlink(filename_.c_str());
+  TempFile(const std::string & fn) : filename(fn), tempFileHandle(-1) {
+    strncpy(fn_buffer, fn.c_str(), sizeof(fn_buffer));
+    fn_buffer[sizeof(fn_buffer)-1] = 0;
   }
-
- private:
-  std::string filename_{"/tmp/test_XXXXXXXXXX"};
-  int fd_{-1};
+  ~TempFile() { if(-1 != tempFileHandle) close(tempFileHandle); unlink(fn_buffer); }
 };
 
+
 // Callback to remove a directory in the nftw visitor
 int nftw_remove(const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf)
 {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc
index 147cfee6be82c..01db69d41ef22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_builder_test.cc
@@ -45,7 +45,7 @@ TEST(HdfsBuilderTest, TestRead)
   {
     TempDir tempDir1;
     TempFile tempFile1(tempDir1.path + "/core-site.xml");
-    writeSimpleConfig(tempFile1.GetFileName(), "key1", "value1");
+    writeSimpleConfig(tempFile1.filename, "key1", "value1");
 
     hdfsBuilder * builder = hdfsNewBuilderFromDirectory(tempDir1.path.c_str());
 
@@ -68,7 +68,7 @@ TEST(HdfsBuilderTest, TestRead)
   {
     TempDir tempDir1;
     TempFile tempFile1(tempDir1.path + "/core-site.xml");
-    writeSimpleConfig(tempFile1.GetFileName(), "key1", "100");
+    writeSimpleConfig(tempFile1.filename, "key1", "100");
 
     hdfsBuilder * builder = hdfsNewBuilderFromDirectory(tempDir1.path.c_str());
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
index 4e1bc3b65aec9..b21725c50fae7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_configuration_test.cc
@@ -72,9 +72,9 @@ TEST(HdfsConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
+    writeSimpleConfig(coreSite.filename, "key1", "value1");
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeSimpleConfig(hdfsSite.GetFileName(), "key2", "value2");
+    writeSimpleConfig(hdfsSite.filename, "key2", "value2");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
@@ -89,7 +89,7 @@ TEST(HdfsConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
+    writeSimpleConfig(coreSite.filename, "key1", "value1");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
@@ -103,7 +103,7 @@ TEST(HdfsConfigurationTest, TestDefaultConfigs) {
   {
     TempDir tempDir;
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeSimpleConfig(hdfsSite.GetFileName(), "key2", "value2");
+    writeSimpleConfig(hdfsSite.filename, "key2", "value2");
 
     ConfigurationLoader loader;
     loader.SetSearchPath(tempDir.path);
@@ -121,9 +121,9 @@ TEST(HdfsConfigurationTest, TestConfigParserAPI) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
+    writeSimpleConfig(coreSite.filename, "key1", "value1");
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeSimpleConfig(hdfsSite.GetFileName(), "key2", "value2");
+    writeSimpleConfig(hdfsSite.filename, "key2", "value2");
 
     ConfigParser parser(tempDir.path);
 
@@ -142,9 +142,9 @@ TEST(HdfsConfigurationTest, TestConfigParserAPI) {
   {
     TempDir tempDir;
     TempFile coreSite(tempDir.path + "/core-site.xml");
-    writeSimpleConfig(coreSite.GetFileName(), "key1", "value1");
+    writeSimpleConfig(coreSite.filename, "key1", "value1");
     TempFile hdfsSite(tempDir.path + "/hdfs-site.xml");
-    writeDamagedConfig(hdfsSite.GetFileName(), "key2", "value2");
+    writeDamagedConfig(hdfsSite.filename, "key2", "value2");
 
     ConfigParser parser(tempDir.path);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
index a7847e1db3547..7fa3971cf7edd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
@@ -85,21 +85,3 @@ TEST(XPlatformSyscall, StringCompareIgnoreCaseNegative) {
   EXPECT_FALSE(XPlatform::Syscall::StringCompareIgnoreCase("abcd", "abcde"));
   EXPECT_FALSE(XPlatform::Syscall::StringCompareIgnoreCase("12345", "abcde"));
 }
-
-TEST(XPlatformSyscall, CreateAndOpenTempFileBasic) {
-  std::string pattern("tmp-XXXXXX");
-  std::vector<char> pattern_vec(pattern.begin(), pattern.end());
-
-  const auto fd = XPlatform::Syscall::CreateAndOpenTempFile(pattern_vec);
-  EXPECT_GT(fd, -1);
-  EXPECT_TRUE(XPlatform::Syscall::CloseFile(fd));
-}
-
-TEST(XPlatformSyscall, CreateAndOpenTempFileNegative) {
-  std::string pattern("does-not-adhere-to-pattern");
-  std::vector<char> pattern_vec(pattern.begin(), pattern.end());
-
-  const auto fd = XPlatform::Syscall::CreateAndOpenTempFile(pattern_vec);
-  EXPECT_EQ(fd, -1);
-  EXPECT_FALSE(XPlatform::Syscall::CloseFile(fd));
-}

From af0448d37bd8f303e8db845ecbc2ca31d79a70f7 Mon Sep 17 00:00:00 2001
From: litao <tomleescut@gmail.com>
Date: Sat, 17 Apr 2021 15:49:09 +0800
Subject: [PATCH 36/43] HDFS-15975. Use LongAdder instead of AtomicLong (#2907)

---
 .../metrics2/lib/MutableCounterLong.java      | 10 +++----
 .../hadoop/hdfs/DFSHedgedReadMetrics.java     | 14 +++++-----
 .../hadoop/hdfs/DFSOpsCountStatistics.java    | 20 ++++++-------
 .../fsdataset/impl/FsDatasetCache.java        | 28 +++++++++----------
 .../fsdataset/impl/FsDatasetImpl.java         |  2 +-
 .../hdfs/server/namenode/FSEditLog.java       | 10 +++----
 .../server/namenode/FSNamesystemLock.java     | 13 +++++----
 .../org/apache/hadoop/hdfs/TestPread.java     |  8 +++---
 8 files changed, 53 insertions(+), 52 deletions(-)

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java
index d3dec2e4d06e2..efaf8a14eaf42 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 
 /**
  * A mutable long counter
@@ -32,11 +32,11 @@
 @InterfaceStability.Evolving
 public class MutableCounterLong extends MutableCounter {
 
-  private AtomicLong value = new AtomicLong();
+  private final LongAdder value = new LongAdder();
 
   public MutableCounterLong(MetricsInfo info, long initValue) {
     super(info);
-    this.value.set(initValue);
+    this.value.add(initValue);
   }
 
   @Override
@@ -49,12 +49,12 @@ public void incr() {
    * @param delta of the increment
    */
   public void incr(long delta) {
-    value.addAndGet(delta);
+    value.add(delta);
     setChanged();
   }
 
   public long value() {
-    return value.get();
+    return value.longValue();
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSHedgedReadMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSHedgedReadMetrics.java
index 2a228e8d01886..1cd9e82cebb08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSHedgedReadMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSHedgedReadMetrics.java
@@ -19,7 +19,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 
 /**
  * The client-side metrics for hedged read feature.
@@ -28,20 +28,20 @@
  */
 @InterfaceAudience.Private
 public class DFSHedgedReadMetrics {
-  public final AtomicLong hedgedReadOps = new AtomicLong();
-  public final AtomicLong hedgedReadOpsWin = new AtomicLong();
-  public final AtomicLong hedgedReadOpsInCurThread = new AtomicLong();
+  public final LongAdder hedgedReadOps = new LongAdder();
+  public final LongAdder hedgedReadOpsWin = new LongAdder();
+  public final LongAdder hedgedReadOpsInCurThread = new LongAdder();
 
   public void incHedgedReadOps() {
-    hedgedReadOps.incrementAndGet();
+    hedgedReadOps.increment();
   }
 
   public void incHedgedReadOpsInCurThread() {
-    hedgedReadOpsInCurThread.incrementAndGet();
+    hedgedReadOpsInCurThread.increment();
   }
 
   public void incHedgedReadWins() {
-    hedgedReadOpsWin.incrementAndGet();
+    hedgedReadOpsWin.increment();
   }
 
   public long getHedgedReadOps() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index fdd0072905fd4..04fef2d5dcd23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -26,7 +26,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.NoSuchElementException;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 
 /**
  * This storage statistics tracks how many times each DFS operation was issued.
@@ -141,21 +141,21 @@ public static OpType fromSymbol(String symbol) {
 
   public static final String NAME = "DFSOpsCountStatistics";
 
-  private final Map<OpType, AtomicLong> opsCount = new EnumMap<>(OpType.class);
+  private final Map<OpType, LongAdder> opsCount = new EnumMap<>(OpType.class);
 
   public DFSOpsCountStatistics() {
     super(NAME);
     for (OpType opType : OpType.values()) {
-      opsCount.put(opType, new AtomicLong(0));
+      opsCount.put(opType, new LongAdder());
     }
   }
 
   public void incrementOpCounter(OpType op) {
-    opsCount.get(op).addAndGet(1);
+    opsCount.get(op).increment();
   }
 
   private class LongIterator implements Iterator<LongStatistic> {
-    private Iterator<Entry<OpType, AtomicLong>> iterator =
+    private final Iterator<Entry<OpType, LongAdder>> iterator =
         opsCount.entrySet().iterator();
 
     @Override
@@ -168,9 +168,9 @@ public LongStatistic next() {
       if (!iterator.hasNext()) {
         throw new NoSuchElementException();
       }
-      final Entry<OpType, AtomicLong> entry = iterator.next();
+      final Entry<OpType, LongAdder> entry = iterator.next();
       return new LongStatistic(entry.getKey().getSymbol(),
-          entry.getValue().get());
+          entry.getValue().longValue());
     }
 
     @Override
@@ -192,7 +192,7 @@ public Iterator<LongStatistic> getLongStatistics() {
   @Override
   public Long getLong(String key) {
     final OpType type = OpType.fromSymbol(key);
-    return type == null ? null : opsCount.get(type).get();
+    return type == null ? null : opsCount.get(type).longValue();
   }
 
   @Override
@@ -202,8 +202,8 @@ public boolean isTracked(String key) {
 
   @Override
   public void reset() {
-    for (AtomicLong count : opsCount.values()) {
-      count.set(0);
+    for (LongAdder count : opsCount.values()) {
+      count.reset();
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
index b6a57fdeffa77..facace28604a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
@@ -42,7 +42,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.time.DurationFormatUtils;
@@ -120,7 +120,7 @@ public boolean shouldAdvertise() {
   private final HashMap<ExtendedBlockId, Value> mappableBlockMap =
       new HashMap<ExtendedBlockId, Value>();
 
-  private final AtomicLong numBlocksCached = new AtomicLong(0);
+  private final LongAdder numBlocksCached = new LongAdder();
 
   private final FsDatasetImpl dataset;
 
@@ -143,11 +143,11 @@ public boolean shouldAdvertise() {
   /**
    * Number of cache commands that could not be completed successfully
    */
-  final AtomicLong numBlocksFailedToCache = new AtomicLong(0);
+  final LongAdder numBlocksFailedToCache = new LongAdder();
   /**
    * Number of uncache commands that could not be completed successfully
    */
-  final AtomicLong numBlocksFailedToUncache = new AtomicLong(0);
+  final LongAdder numBlocksFailedToUncache = new LongAdder();
 
   public FsDatasetCache(FsDatasetImpl dataset) throws IOException {
     this.dataset = dataset;
@@ -204,7 +204,7 @@ public void initCache(String bpid) throws IOException {
       for (Map.Entry<ExtendedBlockId, MappableBlock> entry : entrySet) {
         mappableBlockMap.put(entry.getKey(),
             new Value(keyToMappableBlock.get(entry.getKey()), State.CACHED));
-        numBlocksCached.addAndGet(1);
+        numBlocksCached.increment();
         dataset.datanode.getMetrics().incrBlocksCached(1);
       }
     }
@@ -278,7 +278,7 @@ synchronized void cacheBlock(long blockId, String bpid,
       LOG.debug("Block with id {}, pool {} already exists in the "
               + "FsDatasetCache with state {}", blockId, bpid, prevValue.state
       );
-      numBlocksFailedToCache.incrementAndGet();
+      numBlocksFailedToCache.increment();
       return;
     }
     mappableBlockMap.put(key, new Value(null, State.CACHING));
@@ -301,7 +301,7 @@ synchronized void uncacheBlock(String bpid, long blockId) {
       LOG.debug("Block with id {}, pool {} does not need to be uncached, "
           + "because it is not currently in the mappableBlockMap.", blockId,
           bpid);
-      numBlocksFailedToUncache.incrementAndGet();
+      numBlocksFailedToUncache.increment();
       return;
     }
     switch (prevValue.state) {
@@ -331,7 +331,7 @@ synchronized void uncacheBlock(String bpid, long blockId) {
     default:
       LOG.debug("Block with id {}, pool {} does not need to be uncached, "
           + "because it is in state {}.", blockId, bpid, prevValue.state);
-      numBlocksFailedToUncache.incrementAndGet();
+      numBlocksFailedToUncache.increment();
       break;
     }
   }
@@ -469,7 +469,7 @@ public void run() {
           dataset.datanode.
               getShortCircuitRegistry().processBlockMlockEvent(key);
         }
-        numBlocksCached.addAndGet(1);
+        numBlocksCached.increment();
         dataset.datanode.getMetrics().incrBlocksCached(1);
         success = true;
       } finally {
@@ -482,7 +482,7 @@ public void run() {
           LOG.debug("Caching of {} was aborted.  We are now caching only {} "
                   + "bytes in total.", key, cacheLoader.getCacheUsed());
           IOUtils.closeQuietly(mappableBlock);
-          numBlocksFailedToCache.incrementAndGet();
+          numBlocksFailedToCache.increment();
 
           synchronized (FsDatasetCache.this) {
             mappableBlockMap.remove(key);
@@ -561,7 +561,7 @@ public void run() {
       }
       long newUsedBytes = cacheLoader.
           release(key, value.mappableBlock.getLength());
-      numBlocksCached.addAndGet(-1);
+      numBlocksCached.decrement();
       dataset.datanode.getMetrics().incrBlocksUncached(1);
       if (revocationTimeMs != 0) {
         LOG.debug("Uncaching of {} completed. usedBytes = {}",
@@ -607,15 +607,15 @@ public long getCacheCapacity() {
   }
 
   public long getNumBlocksFailedToCache() {
-    return numBlocksFailedToCache.get();
+    return numBlocksFailedToCache.longValue();
   }
 
   public long getNumBlocksFailedToUncache() {
-    return numBlocksFailedToUncache.get();
+    return numBlocksFailedToUncache.longValue();
   }
 
   public long getNumBlocksCached() {
-    return numBlocksCached.get();
+    return numBlocksCached.longValue();
   }
 
   public synchronized boolean isCached(String bpid, long blockId) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index d06d3cfec6697..d148f774b98b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2463,7 +2463,7 @@ private void cacheBlock(String bpid, long blockId) {
         success = true;
       } finally {
         if (!success) {
-          cacheManager.numBlocksFailedToCache.incrementAndGet();
+          cacheManager.numBlocksFailedToCache.increment();
         }
       }
       blockFileName = info.getBlockURI().toString();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 6b73bbdff436b..79f039bce92c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -28,7 +28,7 @@
 import java.util.EnumSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -183,7 +183,7 @@ private enum State {
   
   // these are statistics counters.
   private long numTransactions;        // number of transactions
-  private final AtomicLong numTransactionsBatchedInSync = new AtomicLong();
+  private final LongAdder numTransactionsBatchedInSync = new LongAdder();
   private long totalTimeTransactions;  // total time for all transactions
   private NameNodeMetrics metrics;
 
@@ -731,7 +731,7 @@ protected void logSync(long mytxid) {
       if (metrics != null) { // Metrics non-null only when used inside name node
         metrics.addSync(elapsed);
         metrics.incrTransactionsBatchedInSync(editsBatchedInSync);
-        numTransactionsBatchedInSync.addAndGet(editsBatchedInSync);
+        numTransactionsBatchedInSync.add(editsBatchedInSync);
       }
       
     } finally {
@@ -771,7 +771,7 @@ private void printStatistics(boolean force) {
         .append(" Total time for transactions(ms): ")
         .append(totalTimeTransactions)
         .append(" Number of transactions batched in Syncs: ")
-        .append(numTransactionsBatchedInSync.get())
+        .append(numTransactionsBatchedInSync.longValue())
         .append(" Number of syncs: ")
         .append(editLogStream.getNumSync())
         .append(" SyncTimes(ms): ")
@@ -1404,7 +1404,7 @@ private void startLogSegment(final long segmentTxId, int layoutVersion)
     
     numTransactions = 0;
     totalTimeTransactions = 0;
-    numTransactionsBatchedInSync.set(0L);
+    numTransactionsBatchedInSync.reset();
 
     // TODO no need to link this back to storage anymore!
     // See HDFS-2174.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index 842c6b3f2df59..b4f479fa93c89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -22,6 +22,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Supplier;
@@ -113,12 +114,12 @@ public Long initialValue() {
    * The number of time the read lock
    * has been held longer than the threshold.
    */
-  private final AtomicLong numReadLockLongHold = new AtomicLong(0);
+  private final LongAdder numReadLockLongHold = new LongAdder();
   /**
    * The number of time the write lock
    * has been held for longer than the threshold.
    */
-  private final AtomicLong numWriteLockLongHold = new AtomicLong(0);
+  private final LongAdder numWriteLockLongHold = new LongAdder();
 
   @VisibleForTesting
   static final String OP_NAME_OTHER = "OTHER";
@@ -192,7 +193,7 @@ public void readUnlock(String opName,
     final long readLockIntervalMs =
         TimeUnit.NANOSECONDS.toMillis(readLockIntervalNanos);
     if (needReport && readLockIntervalMs >= this.readLockReportingThresholdMs) {
-      numReadLockLongHold.incrementAndGet();
+      numReadLockLongHold.increment();
       String lockReportInfo = null;
       boolean done = false;
       while (!done) {
@@ -309,7 +310,7 @@ private void writeUnlock(String opName, boolean suppressWriteLockReport,
     LogAction logAction = LogThrottlingHelper.DO_NOT_LOG;
     if (needReport &&
         writeLockIntervalMs >= this.writeLockReportingThresholdMs) {
-      numWriteLockLongHold.incrementAndGet();
+      numWriteLockLongHold.increment();
       if (longestWriteLockHeldInfo.getIntervalMs() <= writeLockIntervalMs) {
         String lockReportInfo = lockReportInfoSupplier != null ? " (" +
             lockReportInfoSupplier.get() + ")" : "";
@@ -382,7 +383,7 @@ public int getQueueLength() {
    * has been held longer than the threshold
    */
   public long getNumOfReadLockLongHold() {
-    return numReadLockLongHold.get();
+    return numReadLockLongHold.longValue();
   }
 
   /**
@@ -393,7 +394,7 @@ public long getNumOfReadLockLongHold() {
    * has been held longer than the threshold.
    */
   public long getNumOfWriteLockLongHold() {
-    return numWriteLockLongHold.get();
+    return numWriteLockLongHold.longValue();
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
index ac3c122aac89d..c1e0dbb8e630f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
@@ -401,9 +401,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
     DFSClient dfsClient = fileSys.getClient();
     DFSHedgedReadMetrics metrics = dfsClient.getHedgedReadMetrics();
     // Metrics instance is static, so we need to reset counts from prior tests.
-    metrics.hedgedReadOps.set(0);
-    metrics.hedgedReadOpsWin.set(0);
-    metrics.hedgedReadOpsInCurThread.set(0);
+    metrics.hedgedReadOps.reset();
+    metrics.hedgedReadOpsWin.reset();
+    metrics.hedgedReadOpsInCurThread.reset();
 
     try {
       Path file1 = new Path("hedgedReadMaxOut.dat");
@@ -590,7 +590,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
     String filename = "/hedgedReadMaxOut.dat";
     DFSHedgedReadMetrics metrics = dfsClient.getHedgedReadMetrics();
     // Metrics instance is static, so we need to reset counts from prior tests.
-    metrics.hedgedReadOps.set(0);
+    metrics.hedgedReadOps.reset();
     try {
       Path file = new Path(filename);
       output = fileSys.create(file, (short) 2);

From 9a261b0a16d0454acc9d3eb432b3dd99fff6cbf6 Mon Sep 17 00:00:00 2001
From: Viraj Jasani <vjasani@apache.org>
Date: Sat, 17 Apr 2021 17:17:44 +0530
Subject: [PATCH 37/43] HADOOP-17642. Remove appender EventCounter to avoid
 instantiation (#2922)

Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
---
 .../hadoop-common/src/main/conf/log4j.properties          | 8 +-------
 .../src/main/resources/container-log4j.properties         | 7 +------
 2 files changed, 2 insertions(+), 13 deletions(-)

diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 52d2c1ff038e6..5a2ca4d922852 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -20,7 +20,7 @@ hadoop.log.dir=.
 hadoop.log.file=hadoop.log
 
 # Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
+log4j.rootLogger=${hadoop.root.logger}
 
 # Logging Threshold
 log4j.threshold=ALL
@@ -176,12 +176,6 @@ log4j.appender.DNMETRICSRFA.MaxFileSize=64MB
 log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR
 #log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN
 
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
 
 #
 # shuffle connection log from shuffleHandler
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
index cf499b8b46c0a..678e3a74c897a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
@@ -16,7 +16,7 @@ hadoop.root.logger=DEBUG,CLA
 yarn.app.mapreduce.shuffle.logger=${hadoop.root.logger}
 
 # Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
+log4j.rootLogger=${hadoop.root.logger}
 
 # Logging Threshold
 log4j.threshold=ALL
@@ -69,8 +69,3 @@ log4j.additivity.org.apache.hadoop.mapreduce.task.reduce=false
 #
 log4j.logger.org.apache.hadoop.mapred.Merger=${yarn.app.mapreduce.shuffle.logger}
 log4j.additivity.org.apache.hadoop.mapred.Merger=false
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter

From e66d7d91be39ae570ec59516946ffd623f869ffa Mon Sep 17 00:00:00 2001
From: Masatake Iwasaki <iwasakims@apache.org>
Date: Mon, 19 Apr 2021 16:44:16 +0900
Subject: [PATCH 38/43] HADOOP-17635. Update the ubuntu version in the build
 instruction. (#2931)

Reviewed-by: Akira Ajisaka <aajisaka@apache.org>
---
 BUILDING.txt | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/BUILDING.txt b/BUILDING.txt
index 9bbb6dbf891a9..4d18f698b9543 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -51,14 +51,13 @@ Known issues:
   and run your IDE and Docker etc inside that VM.
 
 ----------------------------------------------------------------------------------
-Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
+Installing required packages for clean install of Ubuntu 18.04 LTS Desktop.
+(For Ubuntu 20.04, gcc/g++ and cmake bundled with Ubuntu can be used.
+Refer to  dev-support/docker/Dockerfile):
 
-* Oracle JDK 1.8 (preferred)
-  $ sudo apt-get purge openjdk*
-  $ sudo apt-get install software-properties-common
-  $ sudo add-apt-repository ppa:webupd8team/java
+* Open JDK 1.8
   $ sudo apt-get update
-  $ sudo apt-get install oracle-java8-installer
+  $ sudo apt-get -y install java-8-openjdk
 * Maven
   $ sudo apt-get -y install maven
 * Native libraries
@@ -86,12 +85,13 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
   $ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2
   $ tar --bzip2 -xf boost_1_72_0.tar.bz2 && cd boost_1_72_0
   $ ./bootstrap.sh --prefix=/usr/
-  $ ./b2 --without-python install
+  $ ./b2 --without-python
+  $ sudo ./b2 --without-python install
 
 Optional packages:
 
 * Snappy compression (only used for hadoop-mapreduce-client-nativetask)
-  $ sudo apt-get install snappy libsnappy-dev
+  $ sudo apt-get install libsnappy-dev
 * Intel ISA-L library for erasure coding
   Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
   (OR https://github.com/01org/isa-l)

From 85a3532849d81ca929bc52cb7ca905c4f53652ec Mon Sep 17 00:00:00 2001
From: litao <tomleescut@gmail.com>
Date: Mon, 19 Apr 2021 19:42:30 +0800
Subject: [PATCH 39/43] HDFS-15970. Print network topology on the web (#2896)

---
 .../server/namenode/NameNodeHttpServer.java   |   6 +-
 .../namenode/NetworkTopologyServlet.java      | 187 ++++++++++++++++
 .../src/main/webapps/hdfs/dfshealth.html      |   1 +
 .../src/main/webapps/hdfs/explorer.html       |   1 +
 .../namenode/TestNetworkTopologyServlet.java  | 202 ++++++++++++++++++
 5 files changed, 395 insertions(+), 2 deletions(-)
 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java
 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index 33913227af2ce..7ca52417d9a0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -166,7 +166,7 @@ void start() throws IOException {
 
     httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
     httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
-    setupServlets(httpServer, conf);
+    setupServlets(httpServer);
     httpServer.start();
 
     int connIdx = 0;
@@ -243,7 +243,7 @@ void setAliasMap(InMemoryAliasMap aliasMap) {
     httpServer.setAttribute(ALIASMAP_ATTRIBUTE_KEY, aliasMap);
   }
 
-  private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
+  private static void setupServlets(HttpServer2 httpServer) {
     httpServer.addInternalServlet("startupProgress",
         StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class);
     httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class,
@@ -253,6 +253,8 @@ private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
     httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME,
         IsNameNodeActiveServlet.PATH_SPEC,
         IsNameNodeActiveServlet.class);
+    httpServer.addInternalServlet("topology",
+        NetworkTopologyServlet.PATH_SPEC, NetworkTopologyServlet.class);
   }
 
   static FSImage getFsImageFromContext(ServletContext context) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java
new file mode 100644
index 0000000000000..5d089718ccffe
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.util.StringUtils;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.HttpHeaders;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+/**
+ * A servlet to print out the network topology.
+ */
+@InterfaceAudience.Private
+public class NetworkTopologyServlet extends DfsServlet {
+
+  public static final String PATH_SPEC = "/topology";
+
+  protected static final String FORMAT_JSON = "json";
+  protected static final String FORMAT_TEXT = "text";
+
+  @Override
+  public void doGet(HttpServletRequest request, HttpServletResponse response)
+      throws IOException {
+    final ServletContext context = getServletContext();
+
+    String format = parseAcceptHeader(request);
+    if (FORMAT_TEXT.equals(format)) {
+      response.setContentType("text/plain; charset=UTF-8");
+    } else if (FORMAT_JSON.equals(format)) {
+      response.setContentType("application/json; charset=UTF-8");
+    }
+
+    NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context);
+    BlockManager bm = nn.getNamesystem().getBlockManager();
+    List<Node> leaves = bm.getDatanodeManager().getNetworkTopology()
+        .getLeaves(NodeBase.ROOT);
+
+    try (PrintStream out = new PrintStream(
+            response.getOutputStream(), false, "UTF-8")) {
+      printTopology(out, leaves, format);
+    } catch (Throwable t) {
+      String errMsg = "Print network topology failed. "
+              + StringUtils.stringifyException(t);
+      response.sendError(HttpServletResponse.SC_GONE, errMsg);
+      throw new IOException(errMsg);
+    } finally {
+      response.getOutputStream().close();
+    }
+  }
+
+  /**
+   * Display each rack and the nodes assigned to that rack, as determined
+   * by the NameNode, in a hierarchical manner.  The nodes and racks are
+   * sorted alphabetically.
+   *
+   * @param stream print stream
+   * @param leaves leaves nodes under base scope
+   * @param format the response format
+   */
+  public void printTopology(PrintStream stream, List<Node> leaves,
+      String format) throws BadFormatException, IOException {
+    if (leaves.isEmpty()) {
+      stream.print("No DataNodes");
+      return;
+    }
+
+    // Build a map of rack -> nodes
+    Map<String, TreeSet<String>> tree = new HashMap<>();
+    for(Node dni : leaves) {
+      String location = dni.getNetworkLocation();
+      String name = dni.getName();
+
+      tree.putIfAbsent(location, new TreeSet<>());
+      tree.get(location).add(name);
+    }
+
+    // Sort the racks (and nodes) alphabetically, display in order
+    ArrayList<String> racks = new ArrayList<>(tree.keySet());
+    Collections.sort(racks);
+
+    if (FORMAT_JSON.equals(format)) {
+      printJsonFormat(stream, tree, racks);
+    } else if (FORMAT_TEXT.equals(format)) {
+      printTextFormat(stream, tree, racks);
+    } else {
+      throw new BadFormatException("Bad format: " + format);
+    }
+  }
+
+  private void printJsonFormat(PrintStream stream, Map<String,
+      TreeSet<String>> tree, ArrayList<String> racks) throws IOException {
+    JsonFactory dumpFactory = new JsonFactory();
+    JsonGenerator dumpGenerator = dumpFactory.createGenerator(stream);
+    dumpGenerator.writeStartArray();
+
+    for(String r : racks) {
+      dumpGenerator.writeStartObject();
+      dumpGenerator.writeFieldName(r);
+      TreeSet<String> nodes = tree.get(r);
+      dumpGenerator.writeStartArray();
+
+      for(String n : nodes) {
+        dumpGenerator.writeStartObject();
+        dumpGenerator.writeStringField("ip", n);
+        String hostname = NetUtils.getHostNameOfIP(n);
+        if(hostname != null) {
+          dumpGenerator.writeStringField("hostname", hostname);
+        }
+        dumpGenerator.writeEndObject();
+      }
+      dumpGenerator.writeEndArray();
+      dumpGenerator.writeEndObject();
+    }
+    dumpGenerator.writeEndArray();
+    dumpGenerator.flush();
+
+    if (!dumpGenerator.isClosed()) {
+      dumpGenerator.close();
+    }
+  }
+
+  private void printTextFormat(PrintStream stream, Map<String,
+      TreeSet<String>> tree, ArrayList<String> racks) {
+    for(String r : racks) {
+      stream.println("Rack: " + r);
+      TreeSet<String> nodes = tree.get(r);
+
+      for(String n : nodes) {
+        stream.print("   " + n);
+        String hostname = NetUtils.getHostNameOfIP(n);
+        if(hostname != null) {
+          stream.print(" (" + hostname + ")");
+        }
+        stream.println();
+      }
+      stream.println();
+    }
+  }
+
+  @VisibleForTesting
+  static String parseAcceptHeader(HttpServletRequest request) {
+    String format = request.getHeader(HttpHeaders.ACCEPT);
+    return format != null && format.contains(FORMAT_JSON) ?
+            FORMAT_JSON : FORMAT_TEXT;
+  }
+
+  public static class BadFormatException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public BadFormatException(String msg) {
+      super(msg);
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 6e4eade9566d7..8622e4d3a5681 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -52,6 +52,7 @@
         <li><a href="jmx">Metrics</a></li>
         <li><a href="conf">Configuration</a></li>
         <li><a href="stacks">Process Thread Dump</a></li>
+        <li><a href="topology">Network Topology</a></li>
       </ul>
     </li>
   </ul>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index 73bfbd4527f48..3f0509a229700 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -48,6 +48,7 @@
             <li><a href="jmx">Metrics</a></li>
             <li><a href="conf">Configuration</a></li>
             <li><a href="stacks">Process Thread Dump</a></li>
+            <li><a href="topology">Network Topology</a></li>
           </ul>
         </li>
       </ul>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java
new file mode 100644
index 0000000000000..7796ed4182ee6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.StaticMapping;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestNetworkTopologyServlet {
+
+  @Test
+  public void testPrintTopologyTextFormat() throws IOException {
+    StaticMapping.resetMap();
+    Configuration conf = new HdfsConfiguration();
+    int dataNodesNum = 0;
+    final ArrayList<String> rackList = new ArrayList<String>();
+    for (int i = 0; i < 5; i++) {
+      for (int j = 0; j < 2; j++) {
+        rackList.add("/rack" + i);
+        dataNodesNum++;
+      }
+    }
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+            .numDataNodes(dataNodesNum)
+            .racks(rackList.toArray(new String[rackList.size()]))
+            .build();
+    cluster.waitActive();
+
+    // get http uri
+    String httpUri = cluster.getHttpUri(0);
+
+    // send http request
+    URL url = new URL(httpUri + "/topology");
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setReadTimeout(20000);
+    conn.setConnectTimeout(20000);
+    conn.connect();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+    StringBuilder sb =
+        new StringBuilder("-- Network Topology -- \n");
+    sb.append(out);
+    sb.append("\n-- Network Topology -- ");
+    String topology = sb.toString();
+
+    // assert rack info
+    assertTrue(topology.contains("/rack0"));
+    assertTrue(topology.contains("/rack1"));
+    assertTrue(topology.contains("/rack2"));
+    assertTrue(topology.contains("/rack3"));
+    assertTrue(topology.contains("/rack4"));
+
+    // assert node number
+    assertEquals(topology.split("127.0.0.1").length - 1,
+        dataNodesNum);
+  }
+
+  @Test
+  public void testPrintTopologyJsonFormat() throws IOException {
+      StaticMapping.resetMap();
+      Configuration conf = new HdfsConfiguration();
+      int dataNodesNum = 0;
+      final ArrayList<String> rackList = new ArrayList<String>();
+      for (int i = 0; i < 5; i++) {
+          for (int j = 0; j < 2; j++) {
+              rackList.add("/rack" + i);
+              dataNodesNum++;
+          }
+      }
+
+      MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+              .numDataNodes(dataNodesNum)
+              .racks(rackList.toArray(new String[rackList.size()]))
+              .build();
+      cluster.waitActive();
+
+      // get http uri
+      String httpUri = cluster.getHttpUri(0);
+
+      // send http request
+      URL url = new URL(httpUri + "/topology");
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.setReadTimeout(20000);
+      conn.setConnectTimeout(20000);
+      conn.setRequestProperty("Accept", "application/json");
+      conn.connect();
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+      String topology = out.toString();
+
+      // parse json
+      JsonNode racks = new ObjectMapper().readTree(topology);
+
+      // assert rack number
+      assertEquals(racks.size(), 5);
+
+      // assert node number
+      Iterator<JsonNode> elements = racks.elements();
+      int dataNodesCount = 0;
+      while(elements.hasNext()){
+          JsonNode rack = elements.next();
+          Iterator<Map.Entry<String, JsonNode>> fields = rack.fields();
+          while (fields.hasNext()) {
+              dataNodesCount += fields.next().getValue().size();
+          }
+      }
+      assertEquals(dataNodesCount, dataNodesNum);
+  }
+
+  @Test
+  public void testPrintTopologyNoDatanodesTextFormat() throws IOException {
+    StaticMapping.resetMap();
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+            .numDataNodes(0)
+            .build();
+    cluster.waitActive();
+
+    // get http uri
+    String httpUri = cluster.getHttpUri(0);
+
+    // send http request
+    URL url = new URL(httpUri + "/topology");
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setReadTimeout(20000);
+    conn.setConnectTimeout(20000);
+    conn.connect();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+    StringBuilder sb =
+            new StringBuilder("-- Network Topology -- \n");
+    sb.append(out);
+    sb.append("\n-- Network Topology -- ");
+    String topology = sb.toString();
+
+    // assert node number
+    assertTrue(topology.contains("No DataNodes"));
+  }
+
+    @Test
+    public void testPrintTopologyNoDatanodesJsonFormat() throws IOException {
+        StaticMapping.resetMap();
+        Configuration conf = new HdfsConfiguration();
+        MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+                .numDataNodes(0)
+                .build();
+        cluster.waitActive();
+
+        // get http uri
+        String httpUri = cluster.getHttpUri(0);
+
+        // send http request
+        URL url = new URL(httpUri + "/topology");
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setReadTimeout(20000);
+        conn.setConnectTimeout(20000);
+        conn.setRequestProperty("Accept", "application/json");
+        conn.connect();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+        StringBuilder sb =
+                new StringBuilder("-- Network Topology -- \n");
+        sb.append(out);
+        sb.append("\n-- Network Topology -- ");
+        String topology = sb.toString();
+
+        // assert node number
+        assertTrue(topology.contains("No DataNodes"));
+    }
+}

From 2dd1e040108c359292030b214392d46d14797656 Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Tue, 20 Apr 2021 15:32:01 +0100
Subject: [PATCH 40/43] HADOOP-17641. ITestWasbUriAndConfiguration failing.
 (#2937)

This moves the mock account name --which is required to never exist-- from
"mockAccount"  to an account name containing a static UUID.

Contributed by Steve Loughran.
---
 .../org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index e420dabb5d0da..5d2d5d4afdc3f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -65,7 +65,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
   public static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
   public static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
   public static final String WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN = "fs.azure.wasb.account.name";
-  public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net";
+  public static final String MOCK_ACCOUNT_NAME = "mockAccount-c01112a3-2a23-433e-af2a-e808ea385136.blob.core.windows.net";
   public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX = ".blob.core.windows.net";
   public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX = "\\.blob(\\.preprod)?\\.core\\.windows\\.net";
   public static final String MOCK_CONTAINER_NAME = "mockContainer";

From 6cb90005a7d0651474883ac4e1b6961ef74fe513 Mon Sep 17 00:00:00 2001
From: Eric Badger <ebadger@verizonmedia.com>
Date: Tue, 20 Apr 2021 17:34:49 +0000
Subject: [PATCH 41/43] YARN-10723. Change CS nodes page in UI to support
 custom resource. Contributed by Qi Zhu

---
 .../resourcemanager/webapp/NodesPage.java     | 65 +++++++++++++------
 .../resourcemanager/webapp/TestNodesPage.java | 26 +++++++-
 2 files changed, 69 insertions(+), 22 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 8e8a2610f0ce7..545ae881803ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 
 import java.util.Collection;
+import java.util.Map;
 
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
@@ -90,9 +91,7 @@ protected void render(Block html) {
             .th(".mem", "Phys Mem Used %")
             .th(".vcores", "VCores Used")
             .th(".vcores", "VCores Avail")
-            .th(".vcores", "Phys VCores Used %")
-            .th(".gpus", "GPUs Used")
-            .th(".gpus", "GPUs Avail");
+            .th(".vcores", "Phys VCores Used %");
       } else {
         trbody.th(".containers", "Running Containers (G)")
             .th(".allocationTags", "Allocation Tags")
@@ -102,14 +101,26 @@ protected void render(Block html) {
             .th(".vcores", "VCores Used (G)")
             .th(".vcores", "VCores Avail (G)")
             .th(".vcores", "Phys VCores Used %")
-            .th(".gpus", "GPUs Used (G)")
-            .th(".gpus", "GPUs Avail (G)")
             .th(".containers", "Running Containers (O)")
             .th(".mem", "Mem Used (O)")
             .th(".vcores", "VCores Used (O)")
             .th(".containers", "Queued Containers");
       }
 
+      for (Map.Entry<String, Integer> integerEntry :
+          ResourceUtils.getResourceTypeIndex().entrySet()) {
+        if (integerEntry.getKey().equals(ResourceInformation.MEMORY_URI)
+            || integerEntry.getKey().equals(ResourceInformation.VCORES_URI)) {
+          continue;
+        }
+
+        trbody.th("." + integerEntry.getKey(),
+            integerEntry.getKey() + " " + "Used");
+
+        trbody.th("." + integerEntry.getKey(),
+            integerEntry.getKey() + " " + "Avail");
+      }
+
       TBODY<TABLE<Hamlet>> tbody =
           trbody.th(".nodeManagerVersion", "Version").__().__().tbody();
 
@@ -175,17 +186,7 @@ protected void render(Block html) {
           nodeTableData.append("\",\"<a ").append("href='" + "//" + httpAddress)
               .append("'>").append(httpAddress).append("</a>\",").append("\"");
         }
-        Integer gpuIndex = ResourceUtils.getResourceTypeIndex()
-            .get(ResourceInformation.GPU_URI);
-        long usedGPUs = 0;
-        long availableGPUs = 0;
-        if (gpuIndex != null && info.getUsedResource() != null
-            && info.getAvailableResource() != null) {
-          usedGPUs = info.getUsedResource().getResource()
-              .getResourceValue(ResourceInformation.GPU_URI);
-          availableGPUs = info.getAvailableResource().getResource()
-              .getResourceValue(ResourceInformation.GPU_URI);
-        }
+
         nodeTableData.append("<br title='")
             .append(String.valueOf(info.getLastHealthUpdate())).append("'>")
             .append(Times.format(info.getLastHealthUpdate())).append("\",\"")
@@ -205,10 +206,6 @@ protected void render(Block html) {
             .append(String.valueOf(info.getAvailableVirtualCores()))
             .append("\",\"")
             .append(String.valueOf((int) info.getVcoreUtilization()))
-            .append("\",\"")
-            .append(String.valueOf(usedGPUs))
-            .append("\",\"")
-            .append(String.valueOf(availableGPUs))
             .append("\",\"");
 
         // If opportunistic containers are enabled, add extra fields.
@@ -226,6 +223,34 @@ protected void render(Block html) {
               .append("\",\"");
         }
 
+        for (Map.Entry<String, Integer> integerEntry :
+            ResourceUtils.getResourceTypeIndex().entrySet()) {
+          if (integerEntry.getKey().equals(ResourceInformation.MEMORY_URI)
+              || integerEntry.getKey().equals(ResourceInformation.VCORES_URI)) {
+            continue;
+          }
+
+          long usedCustomResource = 0;
+          long availableCustomResource = 0;
+
+          String resourceName = integerEntry.getKey();
+          Integer index = integerEntry.getValue();
+
+          if (index != null && info.getUsedResource() != null
+              && info.getAvailableResource() != null) {
+            usedCustomResource = info.getUsedResource().getResource()
+                .getResourceValue(resourceName);
+            availableCustomResource = info.getAvailableResource().getResource()
+                .getResourceValue(resourceName);
+
+            nodeTableData
+                .append(usedCustomResource)
+                .append("\",\"")
+                .append(availableCustomResource)
+                .append("\",\"");
+          }
+        }
+
         nodeTableData.append(ni.getNodeManagerVersion())
             .append("\"],\n");
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index 9ab6583b06ce4..dd271fd34d798 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -53,7 +53,7 @@ public class TestNodesPage {
   // Number of Actual Table Headers for NodesPage.NodesBlock might change in
   // future. In that case this value should be adjusted to the new value.
   private final int numberOfThInMetricsTable = 23;
-  private final int numberOfActualTableHeaders = 18;
+  private final int numberOfActualTableHeaders = 16;
   private final int numberOfThForOpportunisticContainers = 4;
 
   private Injector injector;
@@ -119,12 +119,34 @@ public void testNodesBlockRenderForLostNodesWithGPUResources()
         initResourceTypes(ResourceInformation.GPU_URI);
     this.setUpInternal(true);
     try {
-      this.testNodesBlockRenderForLostNodes();
+      // Test gpu as a custom resource.
+      //<th class="yarn io/gpu">
+      //  yarn.io/gpu Used
+      //</th>
+      //<th class="yarn io/gpu">
+      //   yarn.io/gpu Avail
+      //</th>
+      this.testNodesBlockRenderForLostNodesWithGPU();
     } finally {
       ResourceUtils.initializeResourcesFromResourceInformationMap(oldRtMap);
     }
   }
 
+  public void testNodesBlockRenderForLostNodesWithGPU() {
+    NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
+    nodesBlock.set("node.state", "lost");
+    nodesBlock.render();
+    PrintWriter writer = injector.getInstance(PrintWriter.class);
+    WebAppTests.flushOutput(injector);
+
+    Mockito.verify(writer,
+        Mockito.times(numberOfActualTableHeaders
+            + numberOfThInMetricsTable + 2))
+        .print("<th");
+    Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
+        .print("<td");
+  }
+
   @Test
   public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
     NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);

From acb49f2fc3b00f3b008d3a92388a3baed40341e6 Mon Sep 17 00:00:00 2001
From: Viraj Jasani <vjasani@apache.org>
Date: Wed, 21 Apr 2021 16:15:49 +0530
Subject: [PATCH 42/43] HDFS-15989. Split TestBalancer and De-flake
 testMaxIterationTime() (#2923)

Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
Signed-off-by: Takanobu Asanuma <tasanuma@apache.org>
---
 .../hdfs/server/balancer/TestBalancer.java    | 616 ---------------
 .../TestBalancerLongRunningTasks.java         | 746 ++++++++++++++++++
 2 files changed, 746 insertions(+), 616 deletions(-)
 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index f59743fdaa73e..378d877cbf703 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -19,8 +19,6 @@
 
 import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY;
 import static org.apache.hadoop.fs.StorageType.DEFAULT;
-import static org.apache.hadoop.fs.StorageType.RAM_DISK;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
@@ -29,25 +27,18 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_KEYTAB_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BLOCK_PINNING_ENABLED;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
-import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
 
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.junit.AfterClass;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -58,10 +49,8 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.net.InetAddress;
-import java.net.InetSocketAddress;
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
@@ -88,8 +77,6 @@
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DFSUtil;
@@ -109,14 +96,9 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.balancer.Balancer.Cli;
 import org.apache.hadoop.hdfs.server.balancer.Balancer.Result;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithUpgradeDomain;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -187,7 +169,6 @@ public void shutdown() throws Exception {
   static final double CAPACITY_ALLOWED_VARIANCE = 0.005;  // 0.5%
   static final double BALANCE_ALLOWED_VARIANCE = 0.11;    // 10%+delta
   static final int DEFAULT_BLOCK_SIZE = 100;
-  static final int DEFAULT_RAM_DISK_BLOCK_SIZE = 5 * 1024 * 1024;
   private static final Random r = new Random();
 
   static {
@@ -213,20 +194,6 @@ static void initConf(Configuration conf) {
     conf.setInt(DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY, 5*1000);
   }
 
-  static void initConfWithRamDisk(Configuration conf,
-                                  long ramDiskCapacity) {
-    conf.setLong(DFS_BLOCK_SIZE_KEY, DEFAULT_RAM_DISK_BLOCK_SIZE);
-    conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, ramDiskCapacity);
-    conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, 3);
-    conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
-    conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
-    conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, 1);
-    conf.setInt(DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY, 5*1000);
-    LazyPersistTestCase.initCacheManipulator();
-
-    conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
-  }
-
   private final ErasureCodingPolicy ecPolicy =
       StripedFileTestUtil.getDefaultECPolicy();
   private final int dataBlocks = ecPolicy.getNumDataUnits();
@@ -487,160 +454,6 @@ static void waitForBalancer(long totalUsedSpace, long totalCapacity,
     waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, 0);
   }
 
-  /**
-   * Make sure that balancer can't move pinned blocks.
-   * If specified favoredNodes when create file, blocks will be pinned use
-   * sticky bit.
-   * @throws Exception
-   */
-  @Test(timeout=100000)
-  public void testBalancerWithPinnedBlocks() throws Exception {
-    // This test assumes stick-bit based block pin mechanism available only
-    // in Linux/Unix. It can be unblocked on Windows when HDFS-7759 is ready to
-    // provide a different mechanism for Windows.
-    assumeNotWindows();
-
-    final Configuration conf = new HdfsConfiguration();
-    initConf(conf);
-    conf.setBoolean(DFS_DATANODE_BLOCK_PINNING_ENABLED, true);
-
-    long[] capacities =  new long[] { CAPACITY, CAPACITY };
-    String[] hosts = {"host0", "host1"};
-    String[] racks = { RACK0, RACK1 };
-    int numOfDatanodes = capacities.length;
-
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length)
-        .hosts(hosts).racks(racks).simulatedCapacities(capacities).build();
-
-    cluster.waitActive();
-    client = NameNodeProxies.createProxy(conf,
-        cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
-
-    // fill up the cluster to be 80% full
-    long totalCapacity = sum(capacities);
-    long totalUsedSpace = totalCapacity * 8 / 10;
-    InetSocketAddress[] favoredNodes = new InetSocketAddress[numOfDatanodes];
-    for (int i = 0; i < favoredNodes.length; i++) {
-      // DFSClient will attempt reverse lookup. In case it resolves
-      // "127.0.0.1" to "localhost", we manually specify the hostname.
-      int port = cluster.getDataNodes().get(i).getXferAddress().getPort();
-      favoredNodes[i] = new InetSocketAddress(hosts[i], port);
-    }
-
-    DFSTestUtil.createFile(cluster.getFileSystem(0), filePath, false, 1024,
-        totalUsedSpace / numOfDatanodes, DEFAULT_BLOCK_SIZE,
-        (short) numOfDatanodes, 0, false, favoredNodes);
-
-    // start up an empty node with the same capacity
-    cluster.startDataNodes(conf, 1, true, null, new String[] { RACK2 },
-        new long[] { CAPACITY });
-
-    totalCapacity += CAPACITY;
-
-    // run balancer and validate results
-    waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
-
-    // start rebalancing
-    Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
-    int r = Balancer.run(namenodes, BalancerParameters.DEFAULT, conf);
-    assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
-  }
-
-  /**
-   * Verify balancer won't violate the default block placement policy.
-   * @throws Exception
-   */
-  @Test(timeout=100000)
-  public void testRackPolicyAfterBalance() throws Exception {
-    final Configuration conf = new HdfsConfiguration();
-    initConf(conf);
-    long[] capacities =  new long[] { CAPACITY, CAPACITY };
-    String[] hosts = {"host0", "host1"};
-    String[] racks = { RACK0, RACK1 };
-    runBalancerAndVerifyBlockPlacmentPolicy(conf, capacities, hosts, racks,
-        null, CAPACITY, "host2", RACK1, null);
-  }
-
-  /**
-   * Verify balancer won't violate upgrade domain block placement policy.
-   * @throws Exception
-   */
-  @Test(timeout=100000)
-  public void testUpgradeDomainPolicyAfterBalance() throws Exception {
-    final Configuration conf = new HdfsConfiguration();
-    initConf(conf);
-    conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
-        BlockPlacementPolicyWithUpgradeDomain.class,
-        BlockPlacementPolicy.class);
-    long[] capacities =  new long[] { CAPACITY, CAPACITY, CAPACITY };
-    String[] hosts = {"host0", "host1", "host2"};
-    String[] racks = { RACK0, RACK1, RACK1 };
-    String[] UDs = { "ud0", "ud1", "ud2" };
-    runBalancerAndVerifyBlockPlacmentPolicy(conf, capacities, hosts, racks,
-        UDs, CAPACITY, "host3", RACK2, "ud2");
-  }
-
-  private void runBalancerAndVerifyBlockPlacmentPolicy(Configuration conf,
-      long[] capacities, String[] hosts, String[] racks, String[] UDs,
-      long newCapacity, String newHost, String newRack, String newUD)
-          throws Exception {
-    int numOfDatanodes = capacities.length;
-
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length)
-        .hosts(hosts).racks(racks).simulatedCapacities(capacities).build();
-    DatanodeManager dm = cluster.getNamesystem().getBlockManager().
-        getDatanodeManager();
-    if (UDs != null) {
-      for(int i = 0; i < UDs.length; i++) {
-        DatanodeID datanodeId = cluster.getDataNodes().get(i).getDatanodeId();
-        dm.getDatanode(datanodeId).setUpgradeDomain(UDs[i]);
-      }
-    }
-
-    try {
-      cluster.waitActive();
-      client = NameNodeProxies.createProxy(conf,
-          cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
-
-      // fill up the cluster to be 80% full
-      long totalCapacity = sum(capacities);
-      long totalUsedSpace = totalCapacity * 8 / 10;
-
-      final long fileSize = totalUsedSpace / numOfDatanodes;
-      DFSTestUtil.createFile(cluster.getFileSystem(0), filePath, false, 1024,
-          fileSize, DEFAULT_BLOCK_SIZE, (short) numOfDatanodes, 0, false);
-
-      // start up an empty node with the same capacity on the same rack as the
-      // pinned host.
-      cluster.startDataNodes(conf, 1, true, null, new String[] { newRack },
-          new String[] { newHost }, new long[] { newCapacity });
-      if (newUD != null) {
-        DatanodeID newId = cluster.getDataNodes().get(
-            numOfDatanodes).getDatanodeId();
-        dm.getDatanode(newId).setUpgradeDomain(newUD);
-      }
-      totalCapacity += newCapacity;
-
-      // run balancer and validate results
-      waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
-
-      // start rebalancing
-      Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
-      Balancer.run(namenodes, BalancerParameters.DEFAULT, conf);
-      BlockPlacementPolicy placementPolicy =
-          cluster.getNamesystem().getBlockManager().getBlockPlacementPolicy();
-      List<LocatedBlock> locatedBlocks = client.
-          getBlockLocations(fileName, 0, fileSize).getLocatedBlocks();
-      for (LocatedBlock locatedBlock : locatedBlocks) {
-        BlockPlacementStatus status = placementPolicy.verifyBlockPlacement(
-            locatedBlock.getLocations(), numOfDatanodes);
-        assertTrue(status.isPlacementPolicySatisfied());
-      }
-    } finally {
-      cluster.shutdown();
-    }
-  }
-
   /**
    * Wait until balanced: each datanode gives utilization within
    * BALANCE_ALLOWED_VARIANCE of average
@@ -1600,144 +1413,6 @@ public void testBalancerCliWithIncludeListWithPortsInAFile() throws Exception {
         CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), true, true);
   }
 
-
-  @Test(timeout = 100000)
-  public void testMaxIterationTime() throws Exception {
-    final Configuration conf = new HdfsConfiguration();
-    initConf(conf);
-    int blockSize = 10*1024*1024; // 10MB block size
-    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
-    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, blockSize);
-    // limit the worker thread count of Balancer to have only 1 queue per DN
-    conf.setInt(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY, 1);
-    // limit the bandwidth to 4MB per sec to emulate slow block moves
-    conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY,
-        4 * 1024 * 1024);
-    // set client socket timeout to have an IN_PROGRESS notification back from
-    // the DataNode about the copy in every second.
-    conf.setLong(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 2000L);
-    // set max iteration time to 2 seconds to timeout before moving any block
-    conf.setLong(DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_KEY, 2000L);
-    // setup the cluster
-    final long capacity = 10L * blockSize;
-    final long[] dnCapacities = new long[] {capacity, capacity};
-    final short rep = 1;
-    final long seed = 0xFAFAFA;
-    cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(0)
-        .build();
-    try {
-      cluster.getConfiguration(0).setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
-      conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
-      cluster.startDataNodes(conf, 1, true, null, null, dnCapacities);
-      cluster.waitClusterUp();
-      cluster.waitActive();
-      final Path path = new Path("/testMaxIterationTime.dat");
-      DistributedFileSystem fs = cluster.getFileSystem();
-      // fill the DN to 40%
-      DFSTestUtil.createFile(fs, path, 4L * blockSize, rep, seed);
-      // start a new DN
-      cluster.startDataNodes(conf, 1, true, null, null, dnCapacities);
-      cluster.triggerHeartbeats();
-      // setup Balancer and run one iteration
-      List<NameNodeConnector> connectors = Collections.emptyList();
-      try {
-        BalancerParameters bParams = BalancerParameters.DEFAULT;
-        // set maxIdleIterations to 1 for NO_MOVE_PROGRESS to be
-        // reported when there is no block move
-        connectors = NameNodeConnector.newNameNodeConnectors(
-            DFSUtil.getInternalNsRpcUris(conf), Balancer.class.getSimpleName(),
-            Balancer.BALANCER_ID_PATH, conf, 1);
-        for (NameNodeConnector nnc : connectors) {
-          LOG.info("NNC to work on: " + nnc);
-          Balancer b = new Balancer(nnc, bParams, conf);
-          Result r = b.runOneIteration();
-          // Since no block cannot be moved in 2 seconds (i.e.,
-          // 4MB/s * 2s = 8MB < 10MB), NO_MOVE_PROGRESS will be reported.
-          // When a block move is not canceled in 2 seconds properly and then
-          // a block is moved unexpectedly, IN_PROGRESS will be reported.
-          assertEquals("We expect ExitStatus.NO_MOVE_PROGRESS to be reported.",
-              ExitStatus.NO_MOVE_PROGRESS, r.getExitStatus());
-          assertEquals(0, r.getBlocksMoved());
-        }
-      } finally {
-        for (NameNodeConnector nnc : connectors) {
-          IOUtils.cleanupWithLogger(null, nnc);
-        }
-      }
-    } finally {
-      cluster.shutdown(true, true);
-    }
-  }
-
-  /*
-   * Test Balancer with Ram_Disk configured
-   * One DN has two files on RAM_DISK, other DN has no files on RAM_DISK.
-   * Then verify that the balancer does not migrate files on RAM_DISK across DN.
-   */
-  @Test(timeout=300000)
-  public void testBalancerWithRamDisk() throws Exception {
-    final int SEED = 0xFADED;
-    final short REPL_FACT = 1;
-    Configuration conf = new Configuration();
-
-    final int defaultRamDiskCapacity = 10;
-    final long ramDiskStorageLimit =
-      ((long) defaultRamDiskCapacity * DEFAULT_RAM_DISK_BLOCK_SIZE) +
-      (DEFAULT_RAM_DISK_BLOCK_SIZE - 1);
-    final long diskStorageLimit =
-      ((long) defaultRamDiskCapacity * DEFAULT_RAM_DISK_BLOCK_SIZE) +
-      (DEFAULT_RAM_DISK_BLOCK_SIZE - 1);
-
-    initConfWithRamDisk(conf, ramDiskStorageLimit);
-
-    cluster = new MiniDFSCluster
-      .Builder(conf)
-      .numDataNodes(1)
-      .storageCapacities(new long[] { ramDiskStorageLimit, diskStorageLimit })
-      .storageTypes(new StorageType[] { RAM_DISK, DEFAULT })
-      .build();
-
-    cluster.waitActive();
-    // Create few files on RAM_DISK
-    final String METHOD_NAME = GenericTestUtils.getMethodName();
-    final Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
-    final Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
-
-    DistributedFileSystem fs = cluster.getFileSystem();
-    DFSClient client = fs.getClient();
-    DFSTestUtil.createFile(fs, path1, true,
-      DEFAULT_RAM_DISK_BLOCK_SIZE, 4 * DEFAULT_RAM_DISK_BLOCK_SIZE,
-      DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
-    DFSTestUtil.createFile(fs, path2, true,
-      DEFAULT_RAM_DISK_BLOCK_SIZE, 1 * DEFAULT_RAM_DISK_BLOCK_SIZE,
-      DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
-
-    // Sleep for a short time to allow the lazy writer thread to do its job
-    Thread.sleep(6 * 1000);
-
-    // Add another fresh DN with the same type/capacity without files on RAM_DISK
-    StorageType[][] storageTypes = new StorageType[][] {{RAM_DISK, DEFAULT}};
-    long[][] storageCapacities = new long[][]{{ramDiskStorageLimit,
-        diskStorageLimit}};
-    cluster.startDataNodes(conf, REPL_FACT, storageTypes, true, null,
-      null, null, storageCapacities, null, false, false, false, null);
-
-    cluster.triggerHeartbeats();
-    Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
-
-    // Run Balancer
-    final BalancerParameters p = BalancerParameters.DEFAULT;
-    final int r = Balancer.run(namenodes, p, conf);
-
-    // Validate no RAM_DISK block should be moved
-    assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
-
-    // Verify files are still on RAM_DISK
-    DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path1, RAM_DISK);
-    DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path2, RAM_DISK);
-  }
-
   /**
    * Check that the balancer exits when there is an unfinalized upgrade.
    */
@@ -1800,66 +1475,6 @@ public void testBalancerDuringUpgrade() throws Exception {
         Balancer.run(namenodes, p, conf));
   }
 
-  /**
-   * Test special case. Two replicas belong to same block should not in same node.
-   * We have 2 nodes.
-   * We have a block in (DN0,SSD) and (DN1,DISK).
-   * Replica in (DN0,SSD) should not be moved to (DN1,SSD).
-   * Otherwise DN1 has 2 replicas.
-   */
-  @Test(timeout=100000)
-  public void testTwoReplicaShouldNotInSameDN() throws Exception {
-    final Configuration conf = new HdfsConfiguration();
-
-    int blockSize = 5 * 1024 * 1024 ;
-    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
-    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
-    conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
-        1L);
-
-    conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
-
-    int numOfDatanodes =2;
-    cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(2)
-        .racks(new String[]{"/default/rack0", "/default/rack0"})
-        .storagesPerDatanode(2)
-        .storageTypes(new StorageType[][]{
-            {StorageType.SSD, StorageType.DISK},
-            {StorageType.SSD, StorageType.DISK}})
-        .storageCapacities(new long[][]{
-            {100 * blockSize, 20 * blockSize},
-            {20 * blockSize, 100 * blockSize}})
-        .build();
-    cluster.waitActive();
-
-    //set "/bar" directory with ONE_SSD storage policy.
-    DistributedFileSystem fs = cluster.getFileSystem();
-    Path barDir = new Path("/bar");
-    fs.mkdir(barDir,new FsPermission((short)777));
-    fs.setStoragePolicy(barDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
-
-    // Insert 30 blocks. So (DN0,SSD) and (DN1,DISK) are about half full,
-    // and (DN0,SSD) and (DN1,DISK) are about 15% full.
-    long fileLen  = 30 * blockSize;
-    // fooFile has ONE_SSD policy. So
-    // (DN0,SSD) and (DN1,DISK) have 2 replicas belong to same block.
-    // (DN0,DISK) and (DN1,SSD) have 2 replicas belong to same block.
-    Path fooFile = new Path(barDir, "foo");
-    createFile(cluster, fooFile, fileLen, (short) numOfDatanodes, 0);
-    // update space info
-    cluster.triggerHeartbeats();
-
-    BalancerParameters p = BalancerParameters.DEFAULT;
-    Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
-    final int r = Balancer.run(namenodes, p, conf);
-
-    // Replica in (DN0,SSD) was not moved to (DN1,SSD), because (DN1,DISK)
-    // already has one. Otherwise DN1 will have 2 replicas.
-    // For same reason, no replicas were moved.
-    assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
-  }
-
   /**
    * Test running many balancer simultaneously.
    *
@@ -1931,121 +1546,6 @@ public void testManyBalancerSimultaneously() throws Exception {
         ExitStatus.SUCCESS.getExitCode(), exitCode);
   }
 
-  /** Balancer should not move blocks with size < minBlockSize. */
-  @Test(timeout=60000)
-  public void testMinBlockSizeAndSourceNodes() throws Exception {
-    final Configuration conf = new HdfsConfiguration();
-    initConf(conf);
-
-    final short replication = 3;
-    final long[] lengths = {10, 10, 10, 10};
-    final long[] capacities = new long[replication];
-    final long totalUsed = capacities.length * sum(lengths);
-    Arrays.fill(capacities, 1000);
-
-    cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(capacities.length)
-        .simulatedCapacities(capacities)
-        .build();
-    final DistributedFileSystem dfs = cluster.getFileSystem();
-    cluster.waitActive();
-    client = NameNodeProxies.createProxy(conf, dfs.getUri(),
-        ClientProtocol.class).getProxy();
-
-    // fill up the cluster to be 80% full
-    for(int i = 0; i < lengths.length; i++) {
-      final long size = lengths[i];
-      final Path p = new Path("/file" + i + "_size" + size);
-      try(OutputStream out = dfs.create(p)) {
-        for(int j = 0; j < size; j++) {
-          out.write(j);
-        }
-      }
-    }
-
-    // start up an empty node with the same capacity
-    cluster.startDataNodes(conf, capacities.length, true, null, null, capacities);
-    LOG.info("capacities    = " + Arrays.toString(capacities));
-    LOG.info("totalUsedSpace= " + totalUsed);
-    LOG.info("lengths       = " + Arrays.toString(lengths) + ", #=" + lengths.length);
-    waitForHeartBeat(totalUsed, 2*capacities[0]*capacities.length, client, cluster);
-
-    final Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
-
-    { // run Balancer with min-block-size=50
-      final BalancerParameters p = Balancer.Cli.parse(new String[] {
-          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
-          "-threshold", "1"
-      });
-      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
-      assertEquals(p.getThreshold(), 1.0, 0.001);
-
-      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
-      final int r = Balancer.run(namenodes, p, conf);
-      assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
-    }
-
-    conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
-
-    { // run Balancer with empty nodes as source nodes
-      final Set<String> sourceNodes = new HashSet<>();
-      final List<DataNode> datanodes = cluster.getDataNodes();
-      for(int i = capacities.length; i < datanodes.size(); i++) {
-        sourceNodes.add(datanodes.get(i).getDisplayName());
-      }
-      final BalancerParameters p = Balancer.Cli.parse(new String[] {
-          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
-          "-threshold", "1",
-          "-source", StringUtils.join(sourceNodes, ',')
-      });
-      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
-      assertEquals(p.getThreshold(), 1.0, 0.001);
-      assertEquals(p.getSourceNodes(), sourceNodes);
-
-      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
-      final int r = Balancer.run(namenodes, p, conf);
-      assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
-    }
-
-    { // run Balancer with a filled node as a source node
-      final Set<String> sourceNodes = new HashSet<>();
-      final List<DataNode> datanodes = cluster.getDataNodes();
-      sourceNodes.add(datanodes.get(0).getDisplayName());
-      final BalancerParameters p = Balancer.Cli.parse(new String[] {
-          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
-          "-threshold", "1",
-          "-source", StringUtils.join(sourceNodes, ',')
-      });
-      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
-      assertEquals(p.getThreshold(), 1.0, 0.001);
-      assertEquals(p.getSourceNodes(), sourceNodes);
-
-      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
-      final int r = Balancer.run(namenodes, p, conf);
-      assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
-    }
-
-    { // run Balancer with all filled node as source nodes
-      final Set<String> sourceNodes = new HashSet<>();
-      final List<DataNode> datanodes = cluster.getDataNodes();
-      for(int i = 0; i < capacities.length; i++) {
-        sourceNodes.add(datanodes.get(i).getDisplayName());
-      }
-      final BalancerParameters p = Balancer.Cli.parse(new String[] {
-          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
-          "-threshold", "1",
-          "-source", StringUtils.join(sourceNodes, ',')
-      });
-      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
-      assertEquals(p.getThreshold(), 1.0, 0.001);
-      assertEquals(p.getSourceNodes(), sourceNodes);
-
-      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
-      final int r = Balancer.run(namenodes, p, conf);
-      assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
-    }
-  }
-
   public void integrationTestWithStripedFile(Configuration conf) throws Exception {
     initConfWithStripe(conf);
     doTestBalancerWithStripedFile(conf);
@@ -2203,122 +1703,6 @@ void testBalancerRPCDelay(int getBlocksMaxQps) throws Exception {
         getBlocksMaxQps, getBlockCallsPerSecond <= getBlocksMaxQps);
   }
 
-  @Test(timeout = 60000)
-  public void testBalancerWithSortTopNodes() throws Exception {
-    final Configuration conf = new HdfsConfiguration();
-    initConf(conf);
-    conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 30000);
-
-    final long capacity = 1000L;
-    final int diffBetweenNodes = 50;
-
-    // Set up the datanodes with two groups:
-    // 5 over-utilized nodes with 80%, 85%, 90%, 95%, 100% usage
-    // 2 under-utilizaed nodes with 0%, 5% usage
-    // With sortTopNodes option, 100% and 95% used ones will be chosen.
-    final int numOfOverUtilizedDn = 5;
-    final int numOfUnderUtilizedDn = 2;
-    final int totalNumOfDn = numOfOverUtilizedDn + numOfUnderUtilizedDn;
-    final long[] capacityArray = new long[totalNumOfDn];
-    Arrays.fill(capacityArray, capacity);
-
-    cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(totalNumOfDn)
-        .simulatedCapacities(capacityArray)
-        .build();
-
-    cluster.setDataNodesDead();
-
-    List<DataNode> dataNodes = cluster.getDataNodes();
-
-    // Create top used nodes
-    for (int i = 0; i < numOfOverUtilizedDn; i++) {
-      // Bring one node alive
-      DataNodeTestUtils.triggerHeartbeat(dataNodes.get(i));
-      DataNodeTestUtils.triggerBlockReport(dataNodes.get(i));
-      // Create nodes with: 80%, 85%, 90%, 95%, 100%.
-      int capacityForThisDatanode = (int)capacity
-          - diffBetweenNodes * (numOfOverUtilizedDn - i - 1);
-      createFile(cluster, new Path("test_big" + i),
-          capacityForThisDatanode, (short) 1, 0);
-      cluster.setDataNodesDead();
-    }
-
-    // Create under utilized nodes
-    for (int i = numOfUnderUtilizedDn - 1; i >= 0; i--) {
-      int index = i + numOfOverUtilizedDn;
-      // Bring one node alive
-      DataNodeTestUtils.triggerHeartbeat(dataNodes.get(index));
-      DataNodeTestUtils.triggerBlockReport(dataNodes.get(index));
-      // Create nodes with: 5%, 0%
-      int capacityForThisDatanode = diffBetweenNodes * i;
-      createFile(cluster,
-          new Path("test_small" + i),
-          capacityForThisDatanode, (short) 1, 0);
-      cluster.setDataNodesDead();
-    }
-
-    // Bring all nodes alive
-    cluster.triggerHeartbeats();
-    cluster.triggerBlockReports();
-    cluster.waitFirstBRCompleted(0, 6000);
-
-    final BalancerParameters p = Balancer.Cli.parse(new String[] {
-        "-policy", BalancingPolicy.Node.INSTANCE.getName(),
-        "-threshold", "1",
-        "-sortTopNodes"
-    });
-
-    client = NameNodeProxies.createProxy(conf,
-        cluster.getFileSystem(0).getUri(),
-        ClientProtocol.class).getProxy();
-
-    // Set max-size-to-move to small number
-    // so only top two nodes will be chosen in one iteration.
-    conf.setLong(DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY, 99L);
-
-    final Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
-
-    List<NameNodeConnector> connectors = NameNodeConnector
-        .newNameNodeConnectors(namenodes,
-            Balancer.class.getSimpleName(), Balancer.BALANCER_ID_PATH, conf,
-            BalancerParameters.DEFAULT.getMaxIdleIteration());
-    final Balancer b = new Balancer(connectors.get(0), p, conf);
-    Result balancerResult = b.runOneIteration();
-
-    cluster.triggerDeletionReports();
-    cluster.triggerBlockReports();
-    cluster.triggerHeartbeats();
-
-    DatanodeInfo[] datanodeReport = client
-        .getDatanodeReport(DatanodeReportType.ALL);
-
-    long maxUsage = 0;
-    for (int i = 0; i < totalNumOfDn; i++) {
-      maxUsage = Math.max(maxUsage, datanodeReport[i].getDfsUsed());
-    }
-
-    // The 95% usage DN will have 9 blocks of 100B and 1 block of 50B - all for the same file.
-    // The HDFS balancer will choose a block to move from this node randomly. More likely it will
-    // be 100B block. Since 100B is greater than DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY which is 99L,
-    // it will stop here. Total bytes moved from this 95% DN will be 1 block of size 100B.
-    // However, chances are the first block selected to be moved from this 95% DN is the 50B block.
-    // After this block is moved, the total moved size so far would be 50B which is smaller than
-    // DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY (99L), hence it will try to move another block.
-    // The second block will always be of size 100B. So total bytes moved from this 95% DN will be
-    // 2 blocks of size (100B + 50B) 150B.
-    // Hence, overall total blocks moved by HDFS balancer would be either of these 2 options:
-    // a) 2 blocks of total size (100B + 100B)
-    // b) 3 blocks of total size (50B + 100B + 100B)
-    assertTrue("BalancerResult is not as expected. " + balancerResult,
-        (balancerResult.getBytesAlreadyMoved() == 200
-            && balancerResult.getBlocksMoved() == 2)
-            || (balancerResult.getBytesAlreadyMoved() == 250
-            && balancerResult.getBlocksMoved() == 3));
-    // 100% and 95% used nodes will be balanced, so top used will be 900
-    assertEquals(900, maxUsage);
-  }
-
   /**
    * @param args
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java
new file mode 100644
index 0000000000000..0d6300cf05e91
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java
@@ -0,0 +1,746 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.balancer;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithUpgradeDomain;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.fs.StorageType.DEFAULT;
+import static org.apache.hadoop.fs.StorageType.RAM_DISK;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BLOCK_PINNING_ENABLED;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC;
+import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Some long running Balancer tasks.
+ */
+public class TestBalancerLongRunningTasks {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestBalancerLongRunningTasks.class);
+
+  static {
+    GenericTestUtils.setLogLevel(Balancer.LOG, Level.TRACE);
+    GenericTestUtils.setLogLevel(Dispatcher.LOG, Level.DEBUG);
+  }
+
+  private final static long CAPACITY = 5000L;
+  private final static String RACK0 = "/rack0";
+  private final static String RACK1 = "/rack1";
+  private final static String RACK2 = "/rack2";
+  private final static String FILE_NAME = "/tmp.txt";
+  private final static Path FILE_PATH = new Path(FILE_NAME);
+  private MiniDFSCluster cluster;
+
+  @After
+  public void shutdown() throws Exception {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  private ClientProtocol client;
+
+  static final int DEFAULT_BLOCK_SIZE = 100;
+  static final int DEFAULT_RAM_DISK_BLOCK_SIZE = 5 * 1024 * 1024;
+
+  static {
+    initTestSetup();
+  }
+
+  public static void initTestSetup() {
+    // do not create id file since it occupies the disk space
+    NameNodeConnector.setWrite2IdFile(false);
+  }
+
+  static void initConf(Configuration conf) {
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE);
+    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
+    conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+        1L);
+    SimulatedFSDataset.setFactory(conf);
+
+    conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
+    conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
+    conf.setInt(DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY, 5 * 1000);
+  }
+
+  static void initConfWithRamDisk(Configuration conf,
+      long ramDiskCapacity) {
+    conf.setLong(DFS_BLOCK_SIZE_KEY, DEFAULT_RAM_DISK_BLOCK_SIZE);
+    conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, ramDiskCapacity);
+    conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, 3);
+    conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
+    conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
+    conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, 1);
+    conf.setInt(DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY, 5 * 1000);
+    LazyPersistTestCase.initCacheManipulator();
+
+    conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
+  }
+
+  /**
+   * Test special case. Two replicas belong to same block should not in same
+   * node.
+   * We have 2 nodes.
+   * We have a block in (DN0,SSD) and (DN1,DISK).
+   * Replica in (DN0,SSD) should not be moved to (DN1,SSD).
+   * Otherwise DN1 has 2 replicas.
+   */
+  @Test(timeout = 100000)
+  public void testTwoReplicaShouldNotInSameDN() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+
+    int blockSize = 5 * 1024 * 1024;
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+    conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+        1L);
+
+    conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
+
+    int numOfDatanodes = 2;
+    cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(2)
+        .racks(new String[]{"/default/rack0", "/default/rack0"})
+        .storagesPerDatanode(2)
+        .storageTypes(new StorageType[][]{
+            {StorageType.SSD, StorageType.DISK},
+            {StorageType.SSD, StorageType.DISK}})
+        .storageCapacities(new long[][]{
+            {100 * blockSize, 20 * blockSize},
+            {20 * blockSize, 100 * blockSize}})
+        .build();
+    cluster.waitActive();
+
+    //set "/bar" directory with ONE_SSD storage policy.
+    DistributedFileSystem fs = cluster.getFileSystem();
+    Path barDir = new Path("/bar");
+    fs.mkdir(barDir, new FsPermission((short) 777));
+    fs.setStoragePolicy(barDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+    // Insert 30 blocks. So (DN0,SSD) and (DN1,DISK) are about half full,
+    // and (DN0,SSD) and (DN1,DISK) are about 15% full.
+    long fileLen = 30 * blockSize;
+    // fooFile has ONE_SSD policy. So
+    // (DN0,SSD) and (DN1,DISK) have 2 replicas belong to same block.
+    // (DN0,DISK) and (DN1,SSD) have 2 replicas belong to same block.
+    Path fooFile = new Path(barDir, "foo");
+    TestBalancer.createFile(cluster, fooFile, fileLen, (short) numOfDatanodes,
+        0);
+    // update space info
+    cluster.triggerHeartbeats();
+
+    BalancerParameters p = BalancerParameters.DEFAULT;
+    Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
+    final int r = Balancer.run(namenodes, p, conf);
+
+    // Replica in (DN0,SSD) was not moved to (DN1,SSD), because (DN1,DISK)
+    // already has one. Otherwise DN1 will have 2 replicas.
+    // For same reason, no replicas were moved.
+    assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+  }
+
+  /*
+   * Test Balancer with Ram_Disk configured
+   * One DN has two files on RAM_DISK, other DN has no files on RAM_DISK.
+   * Then verify that the balancer does not migrate files on RAM_DISK across DN.
+   */
+  @Test(timeout = 300000)
+  public void testBalancerWithRamDisk() throws Exception {
+    final int seed = 0xFADED;
+    final short replicationFactor = 1;
+    Configuration conf = new Configuration();
+
+    final int defaultRamDiskCapacity = 10;
+    final long ramDiskStorageLimit =
+        ((long) defaultRamDiskCapacity * DEFAULT_RAM_DISK_BLOCK_SIZE) +
+            (DEFAULT_RAM_DISK_BLOCK_SIZE - 1);
+    final long diskStorageLimit =
+        ((long) defaultRamDiskCapacity * DEFAULT_RAM_DISK_BLOCK_SIZE) +
+            (DEFAULT_RAM_DISK_BLOCK_SIZE - 1);
+
+    initConfWithRamDisk(conf, ramDiskStorageLimit);
+
+    cluster = new MiniDFSCluster
+        .Builder(conf)
+        .numDataNodes(1)
+        .storageCapacities(new long[]{ramDiskStorageLimit, diskStorageLimit})
+        .storageTypes(new StorageType[]{RAM_DISK, DEFAULT})
+        .build();
+
+    cluster.waitActive();
+    // Create few files on RAM_DISK
+    final String methodName = GenericTestUtils.getMethodName();
+    final Path path1 = new Path("/" + methodName + ".01.dat");
+    final Path path2 = new Path("/" + methodName + ".02.dat");
+
+    DistributedFileSystem fs = cluster.getFileSystem();
+    DFSClient dfsClient = fs.getClient();
+    DFSTestUtil.createFile(fs, path1, true,
+        DEFAULT_RAM_DISK_BLOCK_SIZE, 4 * DEFAULT_RAM_DISK_BLOCK_SIZE,
+        DEFAULT_RAM_DISK_BLOCK_SIZE, replicationFactor, seed, true);
+    DFSTestUtil.createFile(fs, path2, true,
+        DEFAULT_RAM_DISK_BLOCK_SIZE, 1 * DEFAULT_RAM_DISK_BLOCK_SIZE,
+        DEFAULT_RAM_DISK_BLOCK_SIZE, replicationFactor, seed, true);
+
+    // Sleep for a short time to allow the lazy writer thread to do its job
+    Thread.sleep(6 * 1000);
+
+    // Add another fresh DN with the same type/capacity without files on
+    // RAM_DISK
+    StorageType[][] storageTypes = new StorageType[][]{{RAM_DISK, DEFAULT}};
+    long[][] storageCapacities = new long[][]{{ramDiskStorageLimit,
+            diskStorageLimit}};
+    cluster.startDataNodes(conf, replicationFactor, storageTypes, true, null,
+        null, null, storageCapacities, null, false, false, false, null);
+
+    cluster.triggerHeartbeats();
+    Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
+
+    // Run Balancer
+    final BalancerParameters p = BalancerParameters.DEFAULT;
+    final int r = Balancer.run(namenodes, p, conf);
+
+    // Validate no RAM_DISK block should be moved
+    assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+
+    // Verify files are still on RAM_DISK
+    DFSTestUtil.verifyFileReplicasOnStorageType(fs, dfsClient, path1, RAM_DISK);
+    DFSTestUtil.verifyFileReplicasOnStorageType(fs, dfsClient, path2, RAM_DISK);
+  }
+
+  /**
+   * Balancer should not move blocks with size < minBlockSize.
+   */
+  @Test(timeout = 60000)
+  public void testMinBlockSizeAndSourceNodes() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
+
+    final short replication = 3;
+    final long[] lengths = {10, 10, 10, 10};
+    final long[] capacities = new long[replication];
+    final long totalUsed = capacities.length * TestBalancer.sum(lengths);
+    Arrays.fill(capacities, 1000);
+
+    cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(capacities.length)
+        .simulatedCapacities(capacities)
+        .build();
+    final DistributedFileSystem dfs = cluster.getFileSystem();
+    cluster.waitActive();
+    client = NameNodeProxies.createProxy(conf, dfs.getUri(),
+        ClientProtocol.class).getProxy();
+
+    // fill up the cluster to be 80% full
+    for (int i = 0; i < lengths.length; i++) {
+      final long size = lengths[i];
+      final Path p = new Path("/file" + i + "_size" + size);
+      try (OutputStream out = dfs.create(p)) {
+        for (int j = 0; j < size; j++) {
+          out.write(j);
+        }
+      }
+    }
+
+    // start up an empty node with the same capacity
+    cluster.startDataNodes(conf, capacities.length, true, null, null, capacities);
+    LOG.info("capacities    = " + Arrays.toString(capacities));
+    LOG.info("totalUsedSpace= " + totalUsed);
+    LOG.info("lengths       = " + Arrays.toString(lengths) + ", #=" + lengths.length);
+    TestBalancer.waitForHeartBeat(totalUsed,
+        2 * capacities[0] * capacities.length, client, cluster);
+
+    final Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
+
+    { // run Balancer with min-block-size=50
+      final BalancerParameters p = Balancer.Cli.parse(new String[]{
+          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
+          "-threshold", "1"
+      });
+      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
+      assertEquals(p.getThreshold(), 1.0, 0.001);
+
+      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
+      final int r = Balancer.run(namenodes, p, conf);
+      assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+    }
+
+    conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
+
+    { // run Balancer with empty nodes as source nodes
+      final Set<String> sourceNodes = new HashSet<>();
+      final List<DataNode> datanodes = cluster.getDataNodes();
+      for (int i = capacities.length; i < datanodes.size(); i++) {
+        sourceNodes.add(datanodes.get(i).getDisplayName());
+      }
+      final BalancerParameters p = Balancer.Cli.parse(new String[]{
+          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
+          "-threshold", "1",
+          "-source", StringUtils.join(sourceNodes, ',')
+      });
+      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
+      assertEquals(p.getThreshold(), 1.0, 0.001);
+      assertEquals(p.getSourceNodes(), sourceNodes);
+
+      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
+      final int r = Balancer.run(namenodes, p, conf);
+      assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
+    }
+
+    { // run Balancer with a filled node as a source node
+      final Set<String> sourceNodes = new HashSet<>();
+      final List<DataNode> datanodes = cluster.getDataNodes();
+      sourceNodes.add(datanodes.get(0).getDisplayName());
+      final BalancerParameters p = Balancer.Cli.parse(new String[]{
+          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
+          "-threshold", "1",
+          "-source", StringUtils.join(sourceNodes, ',')
+      });
+      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
+      assertEquals(p.getThreshold(), 1.0, 0.001);
+      assertEquals(p.getSourceNodes(), sourceNodes);
+
+      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
+      final int r = Balancer.run(namenodes, p, conf);
+      assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
+    }
+
+    { // run Balancer with all filled node as source nodes
+      final Set<String> sourceNodes = new HashSet<>();
+      final List<DataNode> datanodes = cluster.getDataNodes();
+      for (int i = 0; i < capacities.length; i++) {
+        sourceNodes.add(datanodes.get(i).getDisplayName());
+      }
+      final BalancerParameters p = Balancer.Cli.parse(new String[]{
+          "-policy", BalancingPolicy.Node.INSTANCE.getName(),
+          "-threshold", "1",
+          "-source", StringUtils.join(sourceNodes, ',')
+      });
+      assertEquals(p.getBalancingPolicy(), BalancingPolicy.Node.INSTANCE);
+      assertEquals(p.getThreshold(), 1.0, 0.001);
+      assertEquals(p.getSourceNodes(), sourceNodes);
+
+      conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
+      final int r = Balancer.run(namenodes, p, conf);
+      assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
+    }
+  }
+
+  /**
+   * Verify balancer won't violate upgrade domain block placement policy.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 100000)
+  public void testUpgradeDomainPolicyAfterBalance() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
+    conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+        BlockPlacementPolicyWithUpgradeDomain.class,
+        BlockPlacementPolicy.class);
+    long[] capacities = new long[]{CAPACITY, CAPACITY, CAPACITY};
+    String[] hosts = {"host0", "host1", "host2"};
+    String[] racks = {RACK0, RACK1, RACK1};
+    String[] uds = {"ud0", "ud1", "ud2"};
+    runBalancerAndVerifyBlockPlacmentPolicy(conf, capacities, hosts, racks,
+        uds, CAPACITY, "host3", RACK2, "ud2");
+  }
+
+  /**
+   * Verify balancer won't violate the default block placement policy.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 100000)
+  public void testRackPolicyAfterBalance() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
+    long[] capacities = new long[]{CAPACITY, CAPACITY};
+    String[] hosts = {"host0", "host1"};
+    String[] racks = {RACK0, RACK1};
+    runBalancerAndVerifyBlockPlacmentPolicy(conf, capacities, hosts, racks,
+        null, CAPACITY, "host2", RACK1, null);
+  }
+
+  private void runBalancerAndVerifyBlockPlacmentPolicy(Configuration conf,
+      long[] capacities, String[] hosts, String[] racks, String[] UDs,
+      long newCapacity, String newHost, String newRack, String newUD)
+      throws Exception {
+    int numOfDatanodes = capacities.length;
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length)
+        .hosts(hosts).racks(racks).simulatedCapacities(capacities).build();
+    DatanodeManager dm = cluster.getNamesystem().getBlockManager().
+        getDatanodeManager();
+    if (UDs != null) {
+      for (int i = 0; i < UDs.length; i++) {
+        DatanodeID datanodeId = cluster.getDataNodes().get(i).getDatanodeId();
+        dm.getDatanode(datanodeId).setUpgradeDomain(UDs[i]);
+      }
+    }
+
+    try {
+      cluster.waitActive();
+      client = NameNodeProxies.createProxy(conf,
+          cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
+
+      // fill up the cluster to be 80% full
+      long totalCapacity = TestBalancer.sum(capacities);
+      long totalUsedSpace = totalCapacity * 8 / 10;
+
+      final long fileSize = totalUsedSpace / numOfDatanodes;
+      DFSTestUtil.createFile(cluster.getFileSystem(0), FILE_PATH, false, 1024,
+          fileSize, DEFAULT_BLOCK_SIZE, (short) numOfDatanodes, 0, false);
+
+      // start up an empty node with the same capacity on the same rack as the
+      // pinned host.
+      cluster.startDataNodes(conf, 1, true, null, new String[]{newRack},
+          new String[]{newHost}, new long[]{newCapacity});
+      if (newUD != null) {
+        DatanodeID newId = cluster.getDataNodes().get(
+            numOfDatanodes).getDatanodeId();
+        dm.getDatanode(newId).setUpgradeDomain(newUD);
+      }
+      totalCapacity += newCapacity;
+
+      // run balancer and validate results
+      TestBalancer.waitForHeartBeat(totalUsedSpace,
+          totalCapacity, client, cluster);
+
+      // start rebalancing
+      Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
+      Balancer.run(namenodes, BalancerParameters.DEFAULT, conf);
+      BlockPlacementPolicy placementPolicy =
+          cluster.getNamesystem().getBlockManager().getBlockPlacementPolicy();
+      List<LocatedBlock> locatedBlocks = client.
+          getBlockLocations(FILE_NAME, 0, fileSize).getLocatedBlocks();
+      for (LocatedBlock locatedBlock : locatedBlocks) {
+        BlockPlacementStatus status = placementPolicy.verifyBlockPlacement(
+            locatedBlock.getLocations(), numOfDatanodes);
+        assertTrue(status.isPlacementPolicySatisfied());
+      }
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * Make sure that balancer can't move pinned blocks.
+   * If specified favoredNodes when create file, blocks will be pinned use
+   * sticky bit.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 100000)
+  public void testBalancerWithPinnedBlocks() throws Exception {
+    // This test assumes stick-bit based block pin mechanism available only
+    // in Linux/Unix. It can be unblocked on Windows when HDFS-7759 is ready to
+    // provide a different mechanism for Windows.
+    assumeNotWindows();
+
+    final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
+    conf.setBoolean(DFS_DATANODE_BLOCK_PINNING_ENABLED, true);
+
+    long[] capacities = new long[]{CAPACITY, CAPACITY};
+    String[] hosts = {"host0", "host1"};
+    String[] racks = {RACK0, RACK1};
+    int numOfDatanodes = capacities.length;
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length)
+        .hosts(hosts).racks(racks).simulatedCapacities(capacities).build();
+
+    cluster.waitActive();
+    client = NameNodeProxies.createProxy(conf,
+        cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
+
+    // fill up the cluster to be 80% full
+    long totalCapacity = TestBalancer.sum(capacities);
+    long totalUsedSpace = totalCapacity * 8 / 10;
+    InetSocketAddress[] favoredNodes = new InetSocketAddress[numOfDatanodes];
+    for (int i = 0; i < favoredNodes.length; i++) {
+      // DFSClient will attempt reverse lookup. In case it resolves
+      // "127.0.0.1" to "localhost", we manually specify the hostname.
+      int port = cluster.getDataNodes().get(i).getXferAddress().getPort();
+      favoredNodes[i] = new InetSocketAddress(hosts[i], port);
+    }
+
+    DFSTestUtil.createFile(cluster.getFileSystem(0), FILE_PATH, false, 1024,
+        totalUsedSpace / numOfDatanodes, DEFAULT_BLOCK_SIZE,
+        (short) numOfDatanodes, 0, false, favoredNodes);
+
+    // start up an empty node with the same capacity
+    cluster.startDataNodes(conf, 1, true, null, new String[]{RACK2},
+        new long[]{CAPACITY});
+
+    totalCapacity += CAPACITY;
+
+    // run balancer and validate results
+    TestBalancer.waitForHeartBeat(totalUsedSpace, totalCapacity, client,
+        cluster);
+
+    // start rebalancing
+    Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
+    int r = Balancer.run(namenodes, BalancerParameters.DEFAULT, conf);
+    assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+  }
+
+  @Test(timeout = 60000)
+  public void testBalancerWithSortTopNodes() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
+    conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 30000);
+
+    final long capacity = 1000L;
+    final int diffBetweenNodes = 50;
+
+    // Set up the datanodes with two groups:
+    // 5 over-utilized nodes with 80%, 85%, 90%, 95%, 100% usage
+    // 2 under-utilizaed nodes with 0%, 5% usage
+    // With sortTopNodes option, 100% and 95% used ones will be chosen.
+    final int numOfOverUtilizedDn = 5;
+    final int numOfUnderUtilizedDn = 2;
+    final int totalNumOfDn = numOfOverUtilizedDn + numOfUnderUtilizedDn;
+    final long[] capacityArray = new long[totalNumOfDn];
+    Arrays.fill(capacityArray, capacity);
+
+    cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(totalNumOfDn)
+        .simulatedCapacities(capacityArray)
+        .build();
+
+    cluster.setDataNodesDead();
+
+    List<DataNode> dataNodes = cluster.getDataNodes();
+
+    // Create top used nodes
+    for (int i = 0; i < numOfOverUtilizedDn; i++) {
+      // Bring one node alive
+      DataNodeTestUtils.triggerHeartbeat(dataNodes.get(i));
+      DataNodeTestUtils.triggerBlockReport(dataNodes.get(i));
+      // Create nodes with: 80%, 85%, 90%, 95%, 100%.
+      int capacityForThisDatanode = (int) capacity
+          - diffBetweenNodes * (numOfOverUtilizedDn - i - 1);
+      TestBalancer.createFile(cluster, new Path("test_big" + i),
+          capacityForThisDatanode, (short) 1, 0);
+      cluster.setDataNodesDead();
+    }
+
+    // Create under utilized nodes
+    for (int i = numOfUnderUtilizedDn - 1; i >= 0; i--) {
+      int index = i + numOfOverUtilizedDn;
+      // Bring one node alive
+      DataNodeTestUtils.triggerHeartbeat(dataNodes.get(index));
+      DataNodeTestUtils.triggerBlockReport(dataNodes.get(index));
+      // Create nodes with: 5%, 0%
+      int capacityForThisDatanode = diffBetweenNodes * i;
+      TestBalancer.createFile(cluster,
+          new Path("test_small" + i),
+          capacityForThisDatanode, (short) 1, 0);
+      cluster.setDataNodesDead();
+    }
+
+    // Bring all nodes alive
+    cluster.triggerHeartbeats();
+    cluster.triggerBlockReports();
+    cluster.waitFirstBRCompleted(0, 6000);
+
+    final BalancerParameters p = Balancer.Cli.parse(new String[]{
+        "-policy", BalancingPolicy.Node.INSTANCE.getName(),
+        "-threshold", "1",
+        "-sortTopNodes"
+    });
+
+    client = NameNodeProxies.createProxy(conf,
+        cluster.getFileSystem(0).getUri(),
+        ClientProtocol.class).getProxy();
+
+    // Set max-size-to-move to small number
+    // so only top two nodes will be chosen in one iteration.
+    conf.setLong(DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY, 99L);
+
+    final Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
+
+    List<NameNodeConnector> connectors = NameNodeConnector
+        .newNameNodeConnectors(namenodes,
+            Balancer.class.getSimpleName(), Balancer.BALANCER_ID_PATH, conf,
+            BalancerParameters.DEFAULT.getMaxIdleIteration());
+    final Balancer b = new Balancer(connectors.get(0), p, conf);
+    Balancer.Result balancerResult = b.runOneIteration();
+
+    cluster.triggerDeletionReports();
+    cluster.triggerBlockReports();
+    cluster.triggerHeartbeats();
+
+    DatanodeInfo[] datanodeReport = client
+        .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL);
+
+    long maxUsage = 0;
+    for (int i = 0; i < totalNumOfDn; i++) {
+      maxUsage = Math.max(maxUsage, datanodeReport[i].getDfsUsed());
+    }
+
+    // The 95% usage DN will have 9 blocks of 100B and 1 block of 50B - all for the same file.
+    // The HDFS balancer will choose a block to move from this node randomly. More likely it will
+    // be 100B block. Since 100B is greater than DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY which is 99L,
+    // it will stop here. Total bytes moved from this 95% DN will be 1 block of size 100B.
+    // However, chances are the first block selected to be moved from this 95% DN is the 50B block.
+    // After this block is moved, the total moved size so far would be 50B which is smaller than
+    // DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY (99L), hence it will try to move another block.
+    // The second block will always be of size 100B. So total bytes moved from this 95% DN will be
+    // 2 blocks of size (100B + 50B) 150B.
+    // Hence, overall total blocks moved by HDFS balancer would be either of these 2 options:
+    // a) 2 blocks of total size (100B + 100B)
+    // b) 3 blocks of total size (50B + 100B + 100B)
+    assertTrue("BalancerResult is not as expected. " + balancerResult,
+        (balancerResult.getBytesAlreadyMoved() == 200
+            && balancerResult.getBlocksMoved() == 2)
+            || (balancerResult.getBytesAlreadyMoved() == 250
+            && balancerResult.getBlocksMoved() == 3));
+    // 100% and 95% used nodes will be balanced, so top used will be 900
+    assertEquals(900, maxUsage);
+  }
+
+  @Test(timeout = 100000)
+  public void testMaxIterationTime() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    initConf(conf);
+    int blockSize = 10 * 1024 * 1024; // 10MB block size
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, blockSize);
+    // limit the worker thread count of Balancer to have only 1 queue per DN
+    conf.setInt(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY, 1);
+    // limit the bandwidth to 4MB per sec to emulate slow block moves
+    conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY,
+        4 * 1024 * 1024);
+    // set client socket timeout to have an IN_PROGRESS notification back from
+    // the DataNode about the copy in every second.
+    conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 2000L);
+    // set max iteration time to 500 ms to timeout before moving any block
+    conf.setLong(DFSConfigKeys.DFS_BALANCER_MAX_ITERATION_TIME_KEY, 500L);
+    // setup the cluster
+    final long capacity = 10L * blockSize;
+    final long[] dnCapacities = new long[]{capacity, capacity};
+    final short rep = 1;
+    final long seed = 0xFAFAFA;
+    cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(0)
+        .build();
+    try {
+      cluster.getConfiguration(0).setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+      conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+      cluster.startDataNodes(conf, 1, true, null, null, dnCapacities);
+      cluster.waitClusterUp();
+      cluster.waitActive();
+      final Path path = new Path("/testMaxIterationTime.dat");
+      DistributedFileSystem fs = cluster.getFileSystem();
+      // fill the DN to 40%
+      DFSTestUtil.createFile(fs, path, 4L * blockSize, rep, seed);
+      // start a new DN
+      cluster.startDataNodes(conf, 1, true, null, null, dnCapacities);
+      cluster.triggerHeartbeats();
+      // setup Balancer and run one iteration
+      List<NameNodeConnector> connectors = Collections.emptyList();
+      try {
+        BalancerParameters bParams = BalancerParameters.DEFAULT;
+        // set maxIdleIterations to 1 for NO_MOVE_PROGRESS to be
+        // reported when there is no block move
+        connectors = NameNodeConnector.newNameNodeConnectors(
+            DFSUtil.getInternalNsRpcUris(conf), Balancer.class.getSimpleName(),
+            Balancer.BALANCER_ID_PATH, conf, 1);
+        for (NameNodeConnector nnc : connectors) {
+          LOG.info("NNC to work on: " + nnc);
+          Balancer b = new Balancer(nnc, bParams, conf);
+          Balancer.Result r = b.runOneIteration();
+          // Since no block can be moved in 500 milli-seconds (i.e.,
+          // 4MB/s * 0.5s = 2MB < 10MB), NO_MOVE_PROGRESS will be reported.
+          // When a block move is not canceled in 500 ms properly
+          // (highly unlikely) and then a block is moved unexpectedly,
+          // IN_PROGRESS will be reported. This is highly unlikely unexpected
+          // case. See HDFS-15989.
+          assertEquals("We expect ExitStatus.NO_MOVE_PROGRESS to be reported.",
+              ExitStatus.NO_MOVE_PROGRESS, r.getExitStatus());
+          assertEquals(0, r.getBlocksMoved());
+        }
+      } finally {
+        for (NameNodeConnector nnc : connectors) {
+          IOUtils.cleanupWithLogger(null, nnc);
+        }
+      }
+    } finally {
+      cluster.shutdown(true, true);
+    }
+  }
+
+}

From 5221322b962db2ea95fd422e6fbf6ca4f4009d9e Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Wed, 21 Apr 2021 16:06:06 +0100
Subject: [PATCH 43/43] HADOOP-17535. ABFS: ITestAzureBlobFileSystemCheckAccess
 test failure if no oauth key. (#2920)

Contributed by Steve Loughran.
---
 .../fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java     | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
index 2c7b776ebe411..e52071d92e574 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
@@ -22,7 +22,6 @@
 import java.lang.reflect.Field;
 import java.util.List;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
 import org.junit.Assume;
 import org.junit.Test;
@@ -316,8 +315,8 @@ private void checkPrerequisites() throws Exception {
   private void checkIfConfigIsSet(String configKey){
     AbfsConfiguration conf = getConfiguration();
     String value = conf.get(configKey);
-    Preconditions.checkArgument((value != null && value.trim().length() > 1),
-        configKey + " config is mandatory for the test to run");
+    Assume.assumeTrue(configKey + " config is mandatory for the test to run",
+        value != null && value.trim().length() > 1);
   }
 
   private void assertAccessible(Path testFilePath, FsAction fsAction)