From e3d090b8ddc2adce596206535a9e29def1ee5e9b Mon Sep 17 00:00:00 2001 From: huangzhaobo Date: Sun, 7 Jan 2024 00:55:28 +0800 Subject: [PATCH 1/2] HDFS-17327. Fix reconfig disk balancer parameters for datanode. --- .../org/apache/hadoop/conf/ReconfigurableBase.java | 2 +- .../apache/hadoop/hdfs/server/datanode/DataNode.java | 4 ++-- .../server/datanode/TestDataNodeReconfiguration.java | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java index 1c451ca6d30b9..9d5ae2e2bd3d8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java @@ -228,7 +228,7 @@ public final void reconfigureProperty(String property, String newVal) synchronized(getConf()) { getConf().get(property); String effectiveValue = reconfigurePropertyImpl(property, newVal); - if (newVal != null) { + if (effectiveValue != null) { getConf().set(property, effectiveValue); } else { getConf().unset(property); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 956f5bbe519d4..841eef84bffde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1039,13 +1039,13 @@ private String reconfDiskBalancerParameters(String property, String newVal) result = Boolean.toString(enable); } else if (property.equals(DFS_DISK_BALANCER_PLAN_VALID_INTERVAL)) { if (newVal == null) { - // set to default + // set to the value of the current system or default long defaultInterval = getConf().getTimeDuration( DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, DFS_DISK_BALANCER_PLAN_VALID_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); getDiskBalancer().setPlanValidityInterval(defaultInterval); - result = DFS_DISK_BALANCER_PLAN_VALID_INTERVAL_DEFAULT; + result = Long.toString(defaultInterval); } else { long newInterval = getConf() .getTimeDurationHelper(DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java index 69f0565f2b6b6..44b7db3e46569 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java @@ -916,6 +916,17 @@ public void testDiskBalancerParameters() throws Exception { dn.reconfigureProperty(DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, "1m"); assertEquals(60000, dn.getDiskBalancer().getPlanValidityInterval()); assertEquals(60000, dn.getDiskBalancer().getPlanValidityIntervalInConfig()); + + // Verify set to the value of the current system + long curTimeInterval = dn.getConf().getTimeDuration(DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, + DFS_DISK_BALANCER_PLAN_VALID_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + dn.reconfigureProperty(DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, null); + assertEquals(60000, curTimeInterval); + assertEquals(curTimeInterval, dn.getDiskBalancer().getPlanValidityInterval()); + assertEquals(60000, dn.getDiskBalancer().getPlanValidityIntervalInConfig()); + curTimeInterval = dn.getConf().getTimeDuration(DFS_DISK_BALANCER_PLAN_VALID_INTERVAL, + DFS_DISK_BALANCER_PLAN_VALID_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + assertEquals(60000, curTimeInterval); } } From 2e2769ddf733bbaf154535f92d3d9dd6fa300f5c Mon Sep 17 00:00:00 2001 From: huangzhaobo Date: Sun, 7 Jan 2024 01:18:21 +0800 Subject: [PATCH 2/2] fix log --- .../main/java/org/apache/hadoop/conf/ReconfigurableBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java index 9d5ae2e2bd3d8..f1de0194714de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java @@ -224,11 +224,11 @@ public void shutdownReconfigurationTask() { public final void reconfigureProperty(String property, String newVal) throws ReconfigurationException { if (isPropertyReconfigurable(property)) { - LOG.info("changing property " + property + " to " + newVal); synchronized(getConf()) { getConf().get(property); String effectiveValue = reconfigurePropertyImpl(property, newVal); if (effectiveValue != null) { + LOG.info("changing property " + property + " to " + effectiveValue); getConf().set(property, effectiveValue); } else { getConf().unset(property);