From 1cb2e10ec182e105ad91506392ec49e87d447a88 Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Tue, 25 Jul 2023 18:31:27 +0800 Subject: [PATCH 1/2] HDFS-17123. Sort datanodeStorages when generating StorageBlockReport[] in method BPServiceActor#blockReport for future convenience --- .../hadoop/hdfs/server/protocol/DatanodeStorage.java | 7 ++++++- .../apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 2 +- .../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 3 ++- .../hadoop/hdfs/server/datanode/SimulatedFSDataset.java | 2 +- .../server/datanode/extdataset/ExternalDatasetImpl.java | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java index de4ff252dc0a5..e4042978356b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java @@ -24,7 +24,7 @@ /** * Class captures information of a storage in Datanode. */ -public class DatanodeStorage { +public class DatanodeStorage implements Comparable { /** The state of the storage. */ public enum State { NORMAL, @@ -125,4 +125,9 @@ public boolean equals(Object other){ public int hashCode() { return getStorageID().hashCode(); } + + @Override + public int compareTo(DatanodeStorage that) { + return getStorageID().compareTo(that.getStorageID()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index b552fa277d049..dac42b59d5524 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -412,7 +412,7 @@ List blockReport(long fullBrLeaseId) throws IOException { StorageBlockReport reports[] = new StorageBlockReport[perVolumeBlockLists.size()]; - for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + for (Map.Entry kvPair : perVolumeBlockLists.entrySet()) { BlockListAsLongs blockList = kvPair.getValue(); reports[i++] = new StorageBlockReport(kvPair.getKey(), blockList); totalBlockCount += blockList.getNumberOfBlocks(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index f1115efcc21dc..9740958eb69f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; @@ -2097,7 +2098,7 @@ public List getCacheReport(String bpid) { @Override public Map getBlockReports(String bpid) { Map blockReportsMap = - new HashMap(); + new TreeMap(); Map builders = new HashMap(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 8c75ca9f75208..0ee57bdebb212 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -837,7 +837,7 @@ synchronized BlockListAsLongs getBlockReport(String bpid, @Override public synchronized Map getBlockReports( String bpid) { - Map blockReports = new HashMap<>(); + Map blockReports = new TreeMap(); for (SimulatedStorage storage : storages) { blockReports.put(storage.getDnStorage(), getBlockReport(bpid, storage)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index 1bd42e0bdfbeb..81d8b2f9318b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@ -195,7 +195,7 @@ public void unfinalizeBlock(ExtendedBlock b) throws IOException { @Override public Map getBlockReports(String bpid) { final Map result = - new HashMap(); + new TreeMap(); result.put(storage, BlockListAsLongs.EMPTY); return result; From dd497951b80c563172acfd50a91519fbcd7354cc Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Wed, 26 Jul 2023 23:23:47 +0800 Subject: [PATCH 2/2] fix checkstyles --- .../hadoop/hdfs/server/datanode/SimulatedFSDataset.java | 4 ++-- .../hdfs/server/datanode/extdataset/ExternalDatasetImpl.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 0ee57bdebb212..05e411f7e710b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -26,7 +26,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -837,7 +836,8 @@ synchronized BlockListAsLongs getBlockReport(String bpid, @Override public synchronized Map getBlockReports( String bpid) { - Map blockReports = new TreeMap(); + Map blockReports = + new TreeMap(); for (SimulatedStorage storage : storages) { blockReports.put(storage.getDnStorage(), getBlockReport(bpid, storage)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index 81d8b2f9318b6..11b604f1b1d03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@ -194,8 +194,8 @@ public void unfinalizeBlock(ExtendedBlock b) throws IOException { @Override public Map getBlockReports(String bpid) { - final Map result = - new TreeMap(); + final Map result = + new TreeMap(); result.put(storage, BlockListAsLongs.EMPTY); return result;