From d0d9997e467c98f9c87dcee7a3ceb52de7f74f0a Mon Sep 17 00:00:00 2001 From: Ed Coleman Date: Wed, 24 Apr 2024 15:20:38 +0000 Subject: [PATCH 1/3] Add balancer metrics --- .../core/metrics/MetricsProducer.java | 4 ++ .../org/apache/accumulo/manager/Manager.java | 11 ++++ .../manager/metrics/BalancerMetrics.java | 58 +++++++++++++++++++ .../manager/metrics/ManagerMetrics.java | 4 -- .../accumulo/test/metrics/MetricsIT.java | 3 +- 5 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java index 568b295f580..954e6d98c35 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java @@ -649,6 +649,10 @@ public interface MetricsProducer { String METRICS_UPDATE_WALOG_WRITE = METRICS_UPDATE_PREFIX + "walog.write"; String METRICS_UPDATE_MUTATION_ARRAY_SIZE = METRICS_UPDATE_PREFIX + "mutation.arrays.size"; + // balancer + String METRICS_MANAGER_BALANCER_MIGRATING = "manager.balancer.balancing.total"; + String METRICS_MANAGER_BALANCER_NEED_MIGRATION = "manager.balancer.need.balancing.total"; + /** * Build Micrometer Meter objects and register them with the registry */ diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 86a1dd71d3d..40dbfcedb36 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -112,6 +112,7 @@ import org.apache.accumulo.core.util.Retry; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; +import org.apache.accumulo.manager.metrics.BalancerMetrics; import org.apache.accumulo.manager.metrics.ManagerMetrics; import org.apache.accumulo.manager.recovery.RecoveryManager; import org.apache.accumulo.manager.state.TableCounts; @@ -208,6 +209,7 @@ public class Manager extends AbstractServer private TServer clientService = null; private volatile TabletBalancer tabletBalancer; private final BalancerEnvironment balancerEnvironment; + private final BalancerMetrics balancerMetrics = new BalancerMetrics(); private ManagerState state = ManagerState.INITIAL; @@ -561,6 +563,10 @@ public void clearMigrations(TableId tableId) { } } + public MetricsProducer getBalancerMetrics() { + return balancerMetrics; + } + enum TabletGoalState { HOSTED(TUnloadTabletGoal.UNKNOWN), UNASSIGNED(TUnloadTabletGoal.UNASSIGNED), @@ -934,7 +940,11 @@ private long balanceTablets() { synchronized (balancedNotifier) { balancedNotifier.notifyAll(); } + balancerMetrics.setMigratingCount(0); + balancerMetrics.setNeedMigrationCount(0); } else { + balancerMetrics.setMigratingCount(params.migrationsOut().size()); + balancerMetrics.setNeedMigrationCount(migrations.size()); nextEvent.event("Migrating %d more tablets, %d total", params.migrationsOut().size(), migrations.size()); } @@ -1104,6 +1114,7 @@ public void run() { metricsInfo.addServiceTags(getApplicationName(), sa.getAddress()); var producers = ManagerMetrics.getProducers(getConfiguration(), this); + producers.add(balancerMetrics); metricsInfo.addMetricsProducers(producers.toArray(new MetricsProducer[0])); metricsInfo.init(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java new file mode 100644 index 00000000000..a0631f3b3df --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/BalancerMetrics.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.metrics; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.accumulo.core.metrics.MetricsProducer; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +public class BalancerMetrics implements MetricsProducer { + + AtomicLong migratingCount = new AtomicLong(); + AtomicLong needMigrationCount = new AtomicLong(); + + public long getMigratingCount() { + return migratingCount.get(); + } + + public void setMigratingCount(final long migratingCount) { + this.migratingCount.set(migratingCount); + } + + public long getNeedMigrationCount() { + return needMigrationCount.get(); + } + + public void setNeedMigrationCount(final long needMigrationCount) { + this.needMigrationCount.set(needMigrationCount); + } + + @Override + public void registerMetrics(MeterRegistry registry) { + Gauge.builder(METRICS_MANAGER_BALANCER_MIGRATING, this, BalancerMetrics::getMigratingCount) + .description("Snapshot count of tablets currently being migrated").register(registry); + Gauge + .builder(METRICS_MANAGER_BALANCER_NEED_MIGRATION, this, + BalancerMetrics::getNeedMigrationCount) + .description("Snapshot count of tablets that need to be migrated").register(registry); + } +} diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java index f0502bce988..efe1e6ab017 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java @@ -26,13 +26,9 @@ import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.metrics.fate.FateMetrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ManagerMetrics { - private final static Logger log = LoggerFactory.getLogger(ManagerMetrics.class); - public static List getProducers(AccumuloConfiguration conf, Manager m) { ArrayList producers = new ArrayList<>(); @SuppressWarnings("deprecation") diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java index 4c8c3bea26a..aa7bae035a9 100644 --- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java @@ -101,7 +101,8 @@ public void confirmMetricsPublished() throws Exception { Set unexpectedMetrics = Set.of(METRICS_SCAN_YIELDS, METRICS_UPDATE_ERRORS, METRICS_REPLICATION_QUEUE, METRICS_COMPACTOR_MAJC_STUCK, METRICS_SCAN_BUSY_TIMEOUT); - Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, METRICS_FATE_TYPE_IN_PROGRESS); + Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, METRICS_FATE_TYPE_IN_PROGRESS, + METRICS_MANAGER_BALANCER_NEED_MIGRATION, METRICS_MANAGER_BALANCER_MIGRATING); Map expectedMetricNames = this.getMetricFields(); flakyMetrics.forEach(expectedMetricNames::remove); // might not see these From a92f8cce2a93d1e1e2d25192de3832e98dc71d79 Mon Sep 17 00:00:00 2001 From: Ed Coleman Date: Tue, 7 May 2024 00:11:26 +0000 Subject: [PATCH 2/3] rename balancer metrics (PR suggestion) --- .../org/apache/accumulo/core/metrics/MetricsProducer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java index 3a368ef1c7f..d78549275aa 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java @@ -668,8 +668,8 @@ public interface MetricsProducer { String METRICS_UPDATE_MUTATION_ARRAY_SIZE = METRICS_UPDATE_PREFIX + "mutation.arrays.size"; // balancer - String METRICS_MANAGER_BALANCER_MIGRATING = "manager.balancer.balancing.total"; - String METRICS_MANAGER_BALANCER_NEED_MIGRATION = "manager.balancer.need.balancing.total"; + String METRICS_MANAGER_BALANCER_MIGRATING = "manager.balancer.migrations.in_progress"; + String METRICS_MANAGER_BALANCER_NEED_MIGRATION = "manager.balancer.migrations.needed"; /** * Build Micrometer Meter objects and register them with the registry From d77c9d3ddc7f3979c882b4b0c9a18001f50d107f Mon Sep 17 00:00:00 2001 From: Ed Coleman Date: Tue, 28 May 2024 21:03:00 +0000 Subject: [PATCH 3/3] fix MetricsIT add missing formatter:on --- .../main/java/org/apache/accumulo/test/metrics/MetricsIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java index 2ec4ad191b5..f720e992002 100644 --- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java @@ -116,6 +116,7 @@ public void confirmMetricsPublished() throws Exception { METRICS_SCAN_RESERVATION_TOTAL_TIMER, METRICS_SCAN_RESERVATION_WRITEOUT_TIMER, METRICS_SCAN_TABLET_METADATA_CACHE); + // @formatter:on Map expectedMetricNames = this.getMetricFields(); flakyMetrics.forEach(expectedMetricNames::remove); // might not see these