HDDS-11206. Statistics storage usage indicators include min, max, med…

…ian, stdev (#6977)
apache · Jul 29, 2024 · a532b89 · a532b89
1 parent dcfa3b4
commit a532b89
Show file tree

Hide file tree

Showing 4 changed files with 115 additions and 0 deletions.
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java
@@ -20,6 +20,7 @@
 
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 
+import java.util.HashMap;
 import java.util.Map;
 
 /**
@@ -48,4 +49,8 @@ public interface NodeManagerMXBean {
    */
   Map<String, Map<String, String>> getNodeStatusInfo();
 
+  default Map<String, String> getNodeStatistics() {
+    return new HashMap<>();
+  }
+
 }
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -83,6 +83,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
+import java.util.Arrays;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -1217,6 +1218,53 @@ public static String[] calculateStoragePercentage(
     return storagePercentage;
   }
 
+  @Override
+  public Map<String, String> getNodeStatistics() {
+    Map<String, String> nodeStatistics = new HashMap<>();
+    // Statistics node usaged
+    nodeUsageStatistics(nodeStatistics);
+    // todo: Statistics of other instances
+    return nodeStatistics;
+  }
+
+  private void nodeUsageStatistics(Map<String, String> nodeStatics) {
+    if (nodeStateManager.getAllNodes().size() < 1) {
+      return;
+    }
+    float[] usages = new float[nodeStateManager.getAllNodes().size()];
+    float totalOzoneUsed = 0;
+    int i = 0;
+    for (DatanodeInfo dni : nodeStateManager.getAllNodes()) {
+      String[] storagePercentage = calculateStoragePercentage(
+              dni.getStorageReports());
+      if (storagePercentage[0].equals("N/A")) {
+        usages[i++] = 0;
+      } else {
+        float storagePerc = Float.parseFloat(storagePercentage[0]);
+        usages[i++] = storagePerc;
+        totalOzoneUsed = totalOzoneUsed + storagePerc;
+      }
+    }
+
+    totalOzoneUsed /= nodeStateManager.getAllNodes().size();
+    Arrays.sort(usages);
+    float median = usages[usages.length / 2];
+    nodeStatics.put(UsageStatics.MEDIAN.getLabel(), String.valueOf(median));
+    float max = usages[usages.length - 1];
+    nodeStatics.put(UsageStatics.MAX.getLabel(), String.valueOf(max));
+    float min = usages[0];
+    nodeStatics.put(UsageStatics.MIN.getLabel(), String.valueOf(min));
+
+    float dev = 0;
+    for (i = 0; i < usages.length; i++) {
+      dev += (usages[i] - totalOzoneUsed) * (usages[i] - totalOzoneUsed);
+    }
+    dev = (float) Math.sqrt(dev / usages.length);
+    DecimalFormat decimalFormat = new DecimalFormat("#0.00");
+    decimalFormat.setRoundingMode(RoundingMode.HALF_UP);
+    nodeStatics.put(UsageStatics.STDEV.getLabel(), decimalFormat.format(dev));
+  }
+
   /**
    * Based on the current time and the last heartbeat, calculate the time difference
    * and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc.
@@ -1284,6 +1332,20 @@ public String getLabel() {
     }
   }
 
+  private enum UsageStatics {
+    MIN("Min"),
+    MAX("Max"),
+    MEDIAN("Median"),
+    STDEV("Stdev");
+    private String label;
+    public String getLabel() {
+      return label;
+    }
+    UsageStatics(String label) {
+      this.label = label;
+    }
+  }
+
   /**
    * Returns the min of no healthy volumes reported out of the set
    * of datanodes constituting the pipeline.

diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
@@ -28,6 +28,32 @@ <h2>SCM Information</h2>
     </tbody>
 </table>
 
+<h2>Statistics</h2>
+<table class="table table-bordered table-striped">
+    <tbody>
+    <tr>
+        <th>Datanode Usage</th>
+        <th>in percentage (%)</th>
+    </tr>
+    <tr>
+        <td>Min</td>
+        <td>{{statistics.nodes.usages.min}}</td>
+    </tr>
+    <tr>
+        <td>Median</td>
+        <td>{{statistics.nodes.usages.median}}</td>
+    </tr>
+    <tr>
+        <td>Max</td>
+        <td>{{statistics.nodes.usages.max}}</td>
+    </tr>
+    <tr>
+        <td>Standard Deviation</td>
+        <td>{{statistics.nodes.usages.stdev}}</td>
+    </tr>
+    </tbody>
+</table>
+
 <h2>Node Status</h2>
 <div class="row">
     <div class="col-md-6 text-left">

diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
@@ -32,6 +32,16 @@
             $scope.RecordsToDisplay = "10";
             $scope.currentPage = 1;
             $scope.lastIndex = 0;
+            $scope.statistics = {
+                nodes : {
+                    usages : {
+                        min : "N/A",
+                        max : "N/A",
+                        median : "N/A",
+                        stdev : "N/A"
+                    }
+                }
+            }
 
             function get_protocol(URLScheme, value, baseProto, fallbackProto) {
                 let protocol = "unknown"
@@ -81,6 +91,18 @@
                     $scope.totalItems = nodeStatusCopy.length;
                     $scope.lastIndex = Math.ceil(nodeStatusCopy.length / $scope.RecordsToDisplay);
                     $scope.nodeStatus = nodeStatusCopy.slice(0, $scope.RecordsToDisplay);
+
+                    ctrl.nodemanagermetrics.NodeStatistics.forEach(function(obj) {
+                        if(obj.key == "Min") {
+                            $scope.statistics.nodes.usages.min = obj.value;
+                        } else if(obj.key == "Max") {
+                            $scope.statistics.nodes.usages.max = obj.value;
+                        } else if(obj.key == "Median") {
+                            $scope.statistics.nodes.usages.median = obj.value;
+                        } else if(obj.key == "Stdev") {
+                            $scope.statistics.nodes.usages.stdev = obj.value;
+                        }
+                    });
                 });
             /*if option is 'All' display all records else display specified record on page*/
             $scope.UpdateRecordsToShow = () => {