From 249025331ef6f4e9568b96c18bc0ecfbea92727b Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Fri, 25 Mar 2022 13:19:02 +0530 Subject: [PATCH 1/2] HDDS-6362. Add aggregate metrics to ContainerBalancerMetrics --- .../container/balancer/ContainerBalancer.java | 13 +++-- .../balancer/ContainerBalancerMetrics.java | 50 ++++++++++++++----- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index b2d7d18b2249..f8cfd24725b9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -479,7 +479,7 @@ private void checkIterationMoveResults(Set selectedTargets) { ContainerInfo container = containerManager.getContainer(moveSelection.getContainerID()); this.sizeMovedPerIteration += container.getUsedBytes(); - metrics.incrementNumMovedContainersInLatestIteration(1); + metrics.incrementNumContainerMovesInLatestIteration(1); LOG.info("Move completed for container {} to target {}", container.containerID(), moveSelection.getTargetNode().getUuidString()); @@ -506,10 +506,13 @@ private void checkIterationMoveResults(Set selectedTargets) { sourceToTargetMap.size() + selectedTargets.size(); metrics.incrementNumDatanodesInvolvedInLatestIteration( countDatanodesInvolvedPerIteration); - metrics.incrementDataSizeMovedGBInLatestIteration( - sizeMovedPerIteration / OzoneConsts.GB); + sizeMovedPerIteration /= OzoneConsts.GB; + metrics.incrementDataSizeMovedGBInLatestIteration(sizeMovedPerIteration); + metrics.incrementTotalNumContainerMoves( + metrics.getNumContainerMovesInLatestIteration()); + metrics.incrementTotalSizeMovedGB(sizeMovedPerIteration); LOG.info("Number of datanodes involved in this iteration: {}. Size moved " + - "in this iteration: {}B.", + "in this iteration: {}GB.", countDatanodesInvolvedPerIteration, sizeMovedPerIteration); } @@ -785,7 +788,7 @@ private void resetState() { this.countDatanodesInvolvedPerIteration = 0; this.sizeMovedPerIteration = 0; metrics.resetDataSizeMovedGBInLatestIteration(); - metrics.resetNumMovedContainersInLatestIteration(); + metrics.resetNumContainerMovesInLatestIteration(); metrics.resetNumDatanodesInvolvedInLatestIteration(); metrics.resetDataSizeUnbalancedGB(); metrics.resetNumDatanodesUnbalanced(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java index 07998447ccad..66c513e90bf2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java @@ -40,9 +40,9 @@ public final class ContainerBalancerMetrics { " in the latest iteration.") private MutableCounterLong dataSizeMovedGBInLatestIteration; - @Metric(about = "Number of containers that Container Balancer moved" + - " in the latest iteration.") - private MutableCounterLong numMovedContainersInLatestIteration; + @Metric(about = "Number of container moves performed by Container Balancer " + + "in the latest iteration.") + private MutableCounterLong numContainerMovesInLatestIteration; @Metric(about = "Number of iterations that Container Balancer has run for.") private MutableCounterLong numIterations; @@ -57,6 +57,14 @@ public final class ContainerBalancerMetrics { @Metric(about = "Number of unbalanced datanodes.") private MutableCounterLong numDatanodesUnbalanced; + @Metric(about = "Total number of container moves across all iterations of " + + "Container Balancer.") + private MutableCounterLong totalNumContainerMoves; + + @Metric(about = "Total data size in GB moved across all iterations of " + + "Container Balancer.") + private MutableCounterLong totalSizeMovedGB; + /** * Create and register metrics named {@link ContainerBalancerMetrics#NAME} * for {@link ContainerBalancer}. @@ -92,21 +100,21 @@ public void resetDataSizeMovedGBInLatestIteration() { } /** - * Gets the number of containers moved by Container Balancer in the latest - * iteration. - * @return number of containers + * Gets the number of container moves performed by Container Balancer in the + * latest iteration. + * @return number of container moves */ - public long getNumMovedContainersInLatestIteration() { - return numMovedContainersInLatestIteration.value(); + public long getNumContainerMovesInLatestIteration() { + return numContainerMovesInLatestIteration.value(); } - public void incrementNumMovedContainersInLatestIteration(long valueToAdd) { - this.numMovedContainersInLatestIteration.incr(valueToAdd); + public void incrementNumContainerMovesInLatestIteration(long valueToAdd) { + this.numContainerMovesInLatestIteration.incr(valueToAdd); } - public void resetNumMovedContainersInLatestIteration() { - numMovedContainersInLatestIteration.incr( - -getNumMovedContainersInLatestIteration()); + public void resetNumContainerMovesInLatestIteration() { + numContainerMovesInLatestIteration.incr( + -getNumContainerMovesInLatestIteration()); } /** @@ -170,4 +178,20 @@ public void incrementNumDatanodesUnbalanced(long valueToAdd) { public void resetNumDatanodesUnbalanced() { numDatanodesUnbalanced.incr(-getNumDatanodesUnbalanced()); } + + public long getTotalNumContainerMoves() { + return totalNumContainerMoves.value(); + } + + public void incrementTotalNumContainerMoves(long valueToAdd) { + totalNumContainerMoves.incr(valueToAdd); + } + + public long getTotalSizeMovedGB() { + return totalSizeMovedGB.value(); + } + + public void incrementTotalSizeMovedGB(long valueToAdd) { + totalSizeMovedGB.incr(valueToAdd); + } } From 9e950df2ec4cd9ef449c0911ed015f28e21d97da Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Thu, 31 Mar 2022 18:12:01 +0530 Subject: [PATCH 2/2] rename some metrics --- .../container/balancer/ContainerBalancer.java | 4 ++-- .../balancer/ContainerBalancerMetrics.java | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index f8cfd24725b9..234d7c02f9dc 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -508,9 +508,9 @@ private void checkIterationMoveResults(Set selectedTargets) { countDatanodesInvolvedPerIteration); sizeMovedPerIteration /= OzoneConsts.GB; metrics.incrementDataSizeMovedGBInLatestIteration(sizeMovedPerIteration); - metrics.incrementTotalNumContainerMoves( + metrics.incrementNumContainerMoves( metrics.getNumContainerMovesInLatestIteration()); - metrics.incrementTotalSizeMovedGB(sizeMovedPerIteration); + metrics.incrementDataSizeMovedGB(sizeMovedPerIteration); LOG.info("Number of datanodes involved in this iteration: {}. Size moved " + "in this iteration: {}GB.", countDatanodesInvolvedPerIteration, sizeMovedPerIteration); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java index 66c513e90bf2..f40dd443b14b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java @@ -59,11 +59,11 @@ public final class ContainerBalancerMetrics { @Metric(about = "Total number of container moves across all iterations of " + "Container Balancer.") - private MutableCounterLong totalNumContainerMoves; + private MutableCounterLong numContainerMoves; @Metric(about = "Total data size in GB moved across all iterations of " + "Container Balancer.") - private MutableCounterLong totalSizeMovedGB; + private MutableCounterLong dataSizeMovedGB; /** * Create and register metrics named {@link ContainerBalancerMetrics#NAME} @@ -179,19 +179,19 @@ public void resetNumDatanodesUnbalanced() { numDatanodesUnbalanced.incr(-getNumDatanodesUnbalanced()); } - public long getTotalNumContainerMoves() { - return totalNumContainerMoves.value(); + public long getNumContainerMoves() { + return numContainerMoves.value(); } - public void incrementTotalNumContainerMoves(long valueToAdd) { - totalNumContainerMoves.incr(valueToAdd); + public void incrementNumContainerMoves(long valueToAdd) { + numContainerMoves.incr(valueToAdd); } - public long getTotalSizeMovedGB() { - return totalSizeMovedGB.value(); + public long getDataSizeMovedGB() { + return dataSizeMovedGB.value(); } - public void incrementTotalSizeMovedGB(long valueToAdd) { - totalSizeMovedGB.incr(valueToAdd); + public void incrementDataSizeMovedGB(long valueToAdd) { + dataSizeMovedGB.incr(valueToAdd); } }