HDFS-16376. Expose metrics of NodeNotChosenReason to JMX

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -5057,6 +5057,16 @@ public void shutdown() {
     blocksMap.close();
     MBeans.unregister(mxBeanName);
     mxBeanName = null;
+    BlockPlacementPolicy replicationPolicy =
+        placementPolicies.getPolicy(CONTIGUOUS);
+    if (replicationPolicy != null) {
+      replicationPolicy.clear();
+    }
+    BlockPlacementPolicy ecPolicy =
+        placementPolicies.getPolicy(STRIPED);
+    if (ecPolicy != null) {
+      ecPolicy.clear();
+    }
   }
 
   public void clear() {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementMXBean.java

@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+/**
+ * This is an interface used to retrieve statistic information related to
+ * block placement policy.
+ */
+public interface BlockPlacementMXBean {
+
+  /**
+   * The statistics of why the target nodes are not chosen.
+   *
+   * @return Get the number of reasons why the target nodes are not chosen.
+   */
+  BlockPlacementPolicyDefault.NodeNotChosenReasonMetrics getNumberOfEachNotChosenReason();
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java

@@ -273,4 +273,9 @@ public <T> void splitNodesWithRack(
   public abstract void setExcludeSlowNodesEnabled(boolean enable);
 
   public abstract boolean getExcludeSlowNodesEnabled();
+
+  /**
+   * Clean up resources, such as MxBeans.
+   */
+  public abstract void clear();
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java

@@ -24,8 +24,11 @@
 import static org.apache.hadoop.util.Time.monotonicNow;
 
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
 
+import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -41,6 +44,8 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
+import javax.management.ObjectName;
+
 /**
  * The class is responsible for choosing the desired number of targets
  * for placing block replicas.
@@ -56,7 +61,8 @@
  * rack as the second replica.
  */
 @InterfaceAudience.Private
-public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
+public class BlockPlacementPolicyDefault extends BlockPlacementPolicy
+    implements BlockPlacementMXBean {
 
   private static final String enableDebugLogging =
       "For more information, please enable DEBUG log level on "
@@ -78,7 +84,10 @@ protected StringBuilder initialValue() {
   private static final BlockPlacementStatus ONE_RACK_PLACEMENT =
       new BlockPlacementStatusDefault(1, 1, 1);
 
-  private enum NodeNotChosenReason {
+  private static final ConcurrentHashMap<NodeNotChosenReason, LongAdder> NOT_CHOSEN_REASON_MAP =
+      new ConcurrentHashMap<>();
+
+  public enum NodeNotChosenReason {
     NOT_IN_SERVICE("the node is not in service"),
     NODE_STALE("the node is stale"),
     NODE_TOO_BUSY("the node is too busy"),
@@ -109,7 +118,12 @@ private String getText() {
   private FSClusterStats stats;
   protected long heartbeatInterval;   // interval for DataNode heartbeats
   private long staleInterval;   // interval used to identify stale DataNodes
-
+  private volatile static ObjectName mxBeanName;
+
+  public ObjectName getMxBeanName() {
+    return mxBeanName;
+  }
+
   /**
    * A miss of that many heartbeats is tolerated for replica deletion policy.
    */
@@ -155,6 +169,9 @@ public void initialize(Configuration conf,  FSClusterStats stats,
     this.excludeSlowNodesEnabled = conf.getBoolean(
         DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY,
         DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT);
+    if (mxBeanName == null) {
+      registerMxBeans(this);
+    }
   }
 
   @Override
@@ -988,6 +1005,13 @@ private static void logNodeIsNotChosen(DatanodeDescriptor node,
       base = 0;
     }
     reasonMap.put(reason, base + 1);
+    // To calculate the metrics of NodeNotChosenReason.
+    incrNotChosenReasonNum(reason);
+  }
+
+  private static void incrNotChosenReasonNum(NodeNotChosenReason reason) {
+    NOT_CHOSEN_REASON_MAP.computeIfAbsent(reason, k -> new LongAdder())
+        .increment();
   }
 
   /**
@@ -1369,5 +1393,62 @@ public void setExcludeSlowNodesEnabled(boolean enable) {
   public boolean getExcludeSlowNodesEnabled() {
     return excludeSlowNodesEnabled;
   }
+
+  @Override
+  public NodeNotChosenReasonMetrics getNumberOfEachNotChosenReason() {
+    return new NodeNotChosenReasonMetrics();
+  }
+
+  @Override
+  public void clear() {
+    unregisterMxBeans();
+  }
+
+  private synchronized static void registerMxBeans(Object obj) {
+    mxBeanName = MBeans.register("NameNode", "BlockPlacementStats", obj);
+  }
+
+  private synchronized static void unregisterMxBeans() {
+    MBeans.unregister(mxBeanName);
+    mxBeanName = null;
+  }
+
+  public static class NodeNotChosenReasonMetrics {
+
+    public long getNotInService() {
+      return NOT_CHOSEN_REASON_MAP
+          .getOrDefault(NodeNotChosenReason.NOT_IN_SERVICE, new LongAdder()).longValue();
+    }
+
+    public long getNodeStale() {
+      return NOT_CHOSEN_REASON_MAP
+          .getOrDefault(NodeNotChosenReason.NODE_STALE, new LongAdder()).longValue();
+    }
+
+    public long getNodeTooBusy() {
+      return NOT_CHOSEN_REASON_MAP
+          .getOrDefault(NodeNotChosenReason.NODE_TOO_BUSY, new LongAdder()).longValue();
+    }
+
+    public long getTooManyNodesOnRack() {
+      return NOT_CHOSEN_REASON_MAP
+          .getOrDefault(NodeNotChosenReason.TOO_MANY_NODES_ON_RACK, new LongAdder()).longValue();
+    }
+
+    public long getNotEnoughStorageSpace() {
+      return NOT_CHOSEN_REASON_MAP
+          .getOrDefault(NodeNotChosenReason.NOT_ENOUGH_STORAGE_SPACE, new LongAdder()).longValue();
+    }
+
+    public long getNoRequiredStorageType() {
+      return NOT_CHOSEN_REASON_MAP
+          .getOrDefault(NodeNotChosenReason.NO_REQUIRED_STORAGE_TYPE, new LongAdder()).longValue();
+    }
+
+    public long getNodeSlow() {
+      return NOT_CHOSEN_REASON_MAP
+          .getOrDefault(NodeNotChosenReason.NODE_SLOW, new LongAdder()).longValue();
+    }
+  }
 }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java

@@ -26,6 +26,10 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.openmbean.CompositeDataSupport;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Set;
@@ -122,6 +126,17 @@ public void testChooseTargetExcludeSlowNodes() throws Exception {
         assertTrue(!slowPeers.contains(targets[i].getDatanodeDescriptor()
             .getDatanodeUuid()));
       }
+
+      // Fetch metrics.
+      MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+      ObjectName mxbeanNameFs =
+          new ObjectName("Hadoop:service=NameNode,name=BlockPlacementStats");
+      CompositeDataSupport metrics =
+          (CompositeDataSupport) mbs.getAttribute(mxbeanNameFs,
+              "NumberOfEachNotChosenReason");
+
+      // Assert NodeSlow.
+      assertTrue((long) metrics.get("nodeSlow") > 0);
     } finally {
       namenode.getNamesystem().writeUnlock();
     }