diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 9ec9f9bd47224..624717d887b13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -5057,6 +5057,16 @@ public void shutdown() { blocksMap.close(); MBeans.unregister(mxBeanName); mxBeanName = null; + BlockPlacementPolicy replicationPolicy = + placementPolicies.getPolicy(CONTIGUOUS); + if (replicationPolicy != null) { + replicationPolicy.clear(); + } + BlockPlacementPolicy ecPolicy = + placementPolicies.getPolicy(STRIPED); + if (ecPolicy != null) { + ecPolicy.clear(); + } } public void clear() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementMXBean.java new file mode 100644 index 0000000000000..7c9d074e37fc9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementMXBean.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +/** + * This is an interface used to retrieve statistic information related to + * block placement policy. + */ +public interface BlockPlacementMXBean { + + /** + * The statistics of why the target nodes are not chosen. + * + * @return Get the number of reasons why the target nodes are not chosen. + */ + BlockPlacementPolicyDefault.NodeNotChosenReasonMetrics getNumberOfEachNotChosenReason(); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 68b3bcd5ad6da..cf686719a1937 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -273,4 +273,9 @@ public void splitNodesWithRack( public abstract void setExcludeSlowNodesEnabled(boolean enable); public abstract boolean getExcludeSlowNodesEnabled(); + + /** + * Clean up resources, such as MxBeans. + */ + public abstract void clear(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index dec98d85b52df..f3d5afc9439b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -24,8 +24,11 @@ import static org.apache.hadoop.util.Time.monotonicNow; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; +import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -41,6 +44,8 @@ import org.apache.hadoop.classification.VisibleForTesting; +import javax.management.ObjectName; + /** * The class is responsible for choosing the desired number of targets * for placing block replicas. @@ -56,7 +61,8 @@ * rack as the second replica. */ @InterfaceAudience.Private -public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { +public class BlockPlacementPolicyDefault extends BlockPlacementPolicy + implements BlockPlacementMXBean { private static final String enableDebugLogging = "For more information, please enable DEBUG log level on " @@ -78,7 +84,10 @@ protected StringBuilder initialValue() { private static final BlockPlacementStatus ONE_RACK_PLACEMENT = new BlockPlacementStatusDefault(1, 1, 1); - private enum NodeNotChosenReason { + private static final ConcurrentHashMap NOT_CHOSEN_REASON_MAP = + new ConcurrentHashMap<>(); + + public enum NodeNotChosenReason { NOT_IN_SERVICE("the node is not in service"), NODE_STALE("the node is stale"), NODE_TOO_BUSY("the node is too busy"), @@ -109,7 +118,12 @@ private String getText() { private FSClusterStats stats; protected long heartbeatInterval; // interval for DataNode heartbeats private long staleInterval; // interval used to identify stale DataNodes - + private volatile static ObjectName mxBeanName; + + public ObjectName getMxBeanName() { + return mxBeanName; + } + /** * A miss of that many heartbeats is tolerated for replica deletion policy. */ @@ -155,6 +169,9 @@ public void initialize(Configuration conf, FSClusterStats stats, this.excludeSlowNodesEnabled = conf.getBoolean( DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT); + if (mxBeanName == null) { + registerMxBeans(this); + } } @Override @@ -988,6 +1005,13 @@ private static void logNodeIsNotChosen(DatanodeDescriptor node, base = 0; } reasonMap.put(reason, base + 1); + // To calculate the metrics of NodeNotChosenReason. + incrNotChosenReasonNum(reason); + } + + private static void incrNotChosenReasonNum(NodeNotChosenReason reason) { + NOT_CHOSEN_REASON_MAP.computeIfAbsent(reason, k -> new LongAdder()) + .increment(); } /** @@ -1369,5 +1393,62 @@ public void setExcludeSlowNodesEnabled(boolean enable) { public boolean getExcludeSlowNodesEnabled() { return excludeSlowNodesEnabled; } + + @Override + public NodeNotChosenReasonMetrics getNumberOfEachNotChosenReason() { + return new NodeNotChosenReasonMetrics(); + } + + @Override + public void clear() { + unregisterMxBeans(); + } + + private synchronized static void registerMxBeans(Object obj) { + mxBeanName = MBeans.register("NameNode", "BlockPlacementStats", obj); + } + + private synchronized static void unregisterMxBeans() { + MBeans.unregister(mxBeanName); + mxBeanName = null; + } + + public static class NodeNotChosenReasonMetrics { + + public long getNotInService() { + return NOT_CHOSEN_REASON_MAP + .getOrDefault(NodeNotChosenReason.NOT_IN_SERVICE, new LongAdder()).longValue(); + } + + public long getNodeStale() { + return NOT_CHOSEN_REASON_MAP + .getOrDefault(NodeNotChosenReason.NODE_STALE, new LongAdder()).longValue(); + } + + public long getNodeTooBusy() { + return NOT_CHOSEN_REASON_MAP + .getOrDefault(NodeNotChosenReason.NODE_TOO_BUSY, new LongAdder()).longValue(); + } + + public long getTooManyNodesOnRack() { + return NOT_CHOSEN_REASON_MAP + .getOrDefault(NodeNotChosenReason.TOO_MANY_NODES_ON_RACK, new LongAdder()).longValue(); + } + + public long getNotEnoughStorageSpace() { + return NOT_CHOSEN_REASON_MAP + .getOrDefault(NodeNotChosenReason.NOT_ENOUGH_STORAGE_SPACE, new LongAdder()).longValue(); + } + + public long getNoRequiredStorageType() { + return NOT_CHOSEN_REASON_MAP + .getOrDefault(NodeNotChosenReason.NO_REQUIRED_STORAGE_TYPE, new LongAdder()).longValue(); + } + + public long getNodeSlow() { + return NOT_CHOSEN_REASON_MAP + .getOrDefault(NodeNotChosenReason.NODE_SLOW, new LongAdder()).longValue(); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java index f2c24a646b84a..e0ef803767104 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java @@ -26,6 +26,10 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import javax.management.openmbean.CompositeDataSupport; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Arrays; import java.util.Set; @@ -122,6 +126,17 @@ public void testChooseTargetExcludeSlowNodes() throws Exception { assertTrue(!slowPeers.contains(targets[i].getDatanodeDescriptor() .getDatanodeUuid())); } + + // Fetch metrics. + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanNameFs = + new ObjectName("Hadoop:service=NameNode,name=BlockPlacementStats"); + CompositeDataSupport metrics = + (CompositeDataSupport) mbs.getAttribute(mxbeanNameFs, + "NumberOfEachNotChosenReason"); + + // Assert NodeSlow. + assertTrue((long) metrics.get("nodeSlow") > 0); } finally { namenode.getNamesystem().writeUnlock(); }