diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java index 10c9b1554cf..a66fc0d22fb 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import java.util.HashMap; import java.util.Map; /** @@ -48,4 +49,8 @@ public interface NodeManagerMXBean { */ Map> getNodeStatusInfo(); + default Map getNodeStatistics() { + return new HashMap<>(); + } + } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 818e1826328..038f76b52e9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -83,6 +83,7 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; +import java.util.Arrays; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -1217,6 +1218,53 @@ public static String[] calculateStoragePercentage( return storagePercentage; } + @Override + public Map getNodeStatistics() { + Map nodeStatistics = new HashMap<>(); + // Statistics node usaged + nodeUsageStatistics(nodeStatistics); + // todo: Statistics of other instances + return nodeStatistics; + } + + private void nodeUsageStatistics(Map nodeStatics) { + if (nodeStateManager.getAllNodes().size() < 1) { + return; + } + float[] usages = new float[nodeStateManager.getAllNodes().size()]; + float totalOzoneUsed = 0; + int i = 0; + for (DatanodeInfo dni : nodeStateManager.getAllNodes()) { + String[] storagePercentage = calculateStoragePercentage( + dni.getStorageReports()); + if (storagePercentage[0].equals("N/A")) { + usages[i++] = 0; + } else { + float storagePerc = Float.parseFloat(storagePercentage[0]); + usages[i++] = storagePerc; + totalOzoneUsed = totalOzoneUsed + storagePerc; + } + } + + totalOzoneUsed /= nodeStateManager.getAllNodes().size(); + Arrays.sort(usages); + float median = usages[usages.length / 2]; + nodeStatics.put(UsageStatics.MEDIAN.getLabel(), String.valueOf(median)); + float max = usages[usages.length - 1]; + nodeStatics.put(UsageStatics.MAX.getLabel(), String.valueOf(max)); + float min = usages[0]; + nodeStatics.put(UsageStatics.MIN.getLabel(), String.valueOf(min)); + + float dev = 0; + for (i = 0; i < usages.length; i++) { + dev += (usages[i] - totalOzoneUsed) * (usages[i] - totalOzoneUsed); + } + dev = (float) Math.sqrt(dev / usages.length); + DecimalFormat decimalFormat = new DecimalFormat("#0.00"); + decimalFormat.setRoundingMode(RoundingMode.HALF_UP); + nodeStatics.put(UsageStatics.STDEV.getLabel(), decimalFormat.format(dev)); + } + /** * Based on the current time and the last heartbeat, calculate the time difference * and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc. @@ -1284,6 +1332,20 @@ public String getLabel() { } } + private enum UsageStatics { + MIN("Min"), + MAX("Max"), + MEDIAN("Median"), + STDEV("Stdev"); + private String label; + public String getLabel() { + return label; + } + UsageStatics(String label) { + this.label = label; + } + } + /** * Returns the min of no healthy volumes reported out of the set * of datanodes constituting the pipeline. diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html index be110c9cc56..67655b539f0 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -28,6 +28,32 @@

SCM Information

+

Statistics

+ + + + + + + + + + + + + + + + + + + + + + + +
Datanode Usagein percentage (%)
Min{{statistics.nodes.usages.min}}
Median{{statistics.nodes.usages.median}}
Max{{statistics.nodes.usages.max}}
Standard Deviation{{statistics.nodes.usages.stdev}}
+

Node Status

diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js index c6c79b33d89..8ca9fb257c9 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js @@ -32,6 +32,16 @@ $scope.RecordsToDisplay = "10"; $scope.currentPage = 1; $scope.lastIndex = 0; + $scope.statistics = { + nodes : { + usages : { + min : "N/A", + max : "N/A", + median : "N/A", + stdev : "N/A" + } + } + } function get_protocol(URLScheme, value, baseProto, fallbackProto) { let protocol = "unknown" @@ -81,6 +91,18 @@ $scope.totalItems = nodeStatusCopy.length; $scope.lastIndex = Math.ceil(nodeStatusCopy.length / $scope.RecordsToDisplay); $scope.nodeStatus = nodeStatusCopy.slice(0, $scope.RecordsToDisplay); + + ctrl.nodemanagermetrics.NodeStatistics.forEach(function(obj) { + if(obj.key == "Min") { + $scope.statistics.nodes.usages.min = obj.value; + } else if(obj.key == "Max") { + $scope.statistics.nodes.usages.max = obj.value; + } else if(obj.key == "Median") { + $scope.statistics.nodes.usages.median = obj.value; + } else if(obj.key == "Stdev") { + $scope.statistics.nodes.usages.stdev = obj.value; + } + }); }); /*if option is 'All' display all records else display specified record on page*/ $scope.UpdateRecordsToShow = () => {