From 2e6ab557155d77c2e91c43fefcbe5e4bae4aac08 Mon Sep 17 00:00:00 2001
From: levineliu <levineliu@tencent.com>
Date: Sun, 24 Apr 2022 18:10:17 +0800
Subject: [PATCH] HDFS-16559 Seperate IBR thread waiting time from heart beat
 waiting time.

---
 .../hdfs/server/datanode/BPServiceActor.java   | 18 ++++++++++--------
 .../apache/hadoop/hdfs/TestDatanodeReport.java |  3 ++-
 .../datanode/TestIncrementalBlockReports.java  |  4 ++--
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 838259d7f6a0b..2929cfb844393 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -767,7 +767,10 @@ private void offerService() throws Exception {
 
         // There is no work to do;  sleep until hearbeat timer elapses, 
         // or work arrives, and then iterate again.
-        ibrManager.waitTillNextIBR(scheduler.getHeartbeatWaitTime());
+        final long waitTime = scheduler.getHeartbeatWaitTime();
+        if (waitTime > 0) {
+          sleepAndLogInterrupts(waitTime, "heartbeat interrupted");
+        }
       } catch(RemoteException re) {
         String reClass = re.getClassName();
         if (UnregisteredNodeException.class.getName().equals(reClass) ||
@@ -856,7 +859,7 @@ void register(NamespaceInfo nsInfo) throws IOException {
   }
 
 
-  private void sleepAndLogInterrupts(int millis,
+  private void sleepAndLogInterrupts(long millis,
       String stateString) {
     try {
       Thread.sleep(millis);
@@ -1147,17 +1150,16 @@ public void run() {
       while (shouldRun()) {
         try {
           final long startTime = scheduler.monotonicNow();
-          final boolean sendHeartbeat = scheduler.isHeartbeatDue(startTime);
-          if (!dn.areIBRDisabledForTests() &&
-              (ibrManager.sendImmediately() || sendHeartbeat)) {
+          if (!dn.areIBRDisabledForTests() && ibrManager.sendImmediately()) {
             synchronized (sendIBRLock) {
               ibrManager.sendIBRs(bpNamenode, bpRegistration,
                   bpos.getBlockPoolId(), getRpcMetricSuffix());
             }
           }
-          // There is no work to do; sleep until heartbeat timer elapses,
-          // or work arrives, and then iterate again.
-          ibrManager.waitTillNextIBR(scheduler.getHeartbeatWaitTime());
+          final long endTime = scheduler.monotonicNow();
+          // wait until next ibr is ready to send.
+          // using heart beat interval as max wait time if ibr interval is not configured.
+          ibrManager.waitTillNextIBR(dnConf.heartBeatInterval - (endTime - startTime));
         } catch (Throwable t) {
           LOG.error("Exception in IBRTaskHandler.", t);
           sleepAndLogInterrupts(5000, "offering IBR service");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
index de738eef177a3..f85f73e7fd9c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
@@ -171,7 +171,8 @@ public void testDatanodeReportMissingBlock() throws Exception {
       } catch (IOException e) {
         // all bad datanodes
       }
-      cluster.triggerHeartbeats(); // IBR delete ack
+      // ibr thread is seperated from heart beat thread, use BlockReports to trigger deletion ibr
+      cluster.triggerBlockReports();
       int retries = 0;
       while (true) {
         lb = fs.getClient().getLocatedBlocks(p.toString(), 0).get(0);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
index e848cbfb37ffb..6eb5eb38acb24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
@@ -166,8 +166,8 @@ public void testReportBlockDeleted() throws InterruptedException, IOException {
           anyString(),
           any(StorageReceivedDeletedBlocks[].class));
 
-      // Trigger a heartbeat, this also triggers an IBR.
-      DataNodeTestUtils.triggerHeartbeat(singletonDn);
+      // Trigger a block report, this also triggers an IBR.
+      DataNodeTestUtils.triggerBlockReport(singletonDn);
 
       // Ensure that the deleted block is reported.
       int retries = 0;