diff --git a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java index 413ffba13a890..3aeaa4e65bbe7 100644 --- a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java +++ b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java @@ -54,7 +54,7 @@ public class JvmGcMetrics implements IMetricSet, AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(JvmGcMetrics.class); private final List notificationListenerCleanUpRunnables = new CopyOnWriteArrayList<>(); - private String youngGenPoolName; + private String firstYoungGenPoolName; private String oldGenPoolName; private String nonGenerationalMemoryPool; private final Map lastGcTotalDurationMap = new ConcurrentHashMap<>(); @@ -62,8 +62,8 @@ public class JvmGcMetrics implements IMetricSet, AutoCloseable { public JvmGcMetrics() { for (MemoryPoolMXBean mbean : ManagementFactory.getMemoryPoolMXBeans()) { String name = mbean.getName(); - if (isYoungGenPool(name)) { - youngGenPoolName = name; + if (isFirstYoungGenPool(name)) { + firstYoungGenPoolName = name; } else if (isOldGenPool(name)) { oldGenPoolName = name; } else if (isNonGenerationalHeapPool(name)) { @@ -98,12 +98,8 @@ private static boolean isPartiallyConcurrentGC(GarbageCollectorMXBean gc) { } } - private static boolean isConcurrentPhase(String cause, String name) { - return "No GC".equals(cause) || "Shenandoah Cycles".equals(name); - } - - private static boolean isYoungGenPool(String name) { - return name != null && (name.endsWith("Eden Space") || name.endsWith("Survivor Space")); + private static boolean isFirstYoungGenPool(String name) { + return name != null && name.endsWith("Eden Space"); } private static boolean isOldGenPool(String name) { @@ -142,26 +138,35 @@ public void bindTo(AbstractMetricService metricService) { liveDataSize, AtomicLong::get); - AtomicLong heapMemUsedPercentage = new AtomicLong(calculateMemoryUsagePercentage()); - metricService.createAutoGauge( - SystemMetric.JVM_GC_MEMORY_USED_PERCENT.toString(), - MetricLevel.CORE, - heapMemUsedPercentage, - AtomicLong::get); - - Counter allocatedBytes = - metricService.getOrCreateCounter( - SystemMetric.JVM_GC_MEMORY_ALLOCATED_BYTES.toString(), MetricLevel.CORE); - Counter promotedBytes = (oldGenPoolName == null) ? null : metricService.getOrCreateCounter( SystemMetric.JVM_GC_MEMORY_PROMOTED_BYTES.toString(), MetricLevel.CORE); - // start watching for GC notifications - final AtomicLong heapPoolSizeAfterGc = new AtomicLong(); + Counter nonGenAllocatedBytes = + (nonGenerationalMemoryPool == null) + ? null + : metricService.getOrCreateCounter( + SystemMetric.JVM_GC_NON_GEN_MEMORY_ALLOCATED_BYTES.toString(), MetricLevel.CORE); + Counter oldGenAllocatedBytes = + (oldGenPoolName == null) + ? null + : metricService.getOrCreateCounter( + SystemMetric.JVM_GC_OLD_MEMORY_ALLOCATED_BYTES.toString(), MetricLevel.CORE); + + Counter youngGenAllocatedBytes = + (firstYoungGenPoolName == null) + ? null + : metricService.getOrCreateCounter( + SystemMetric.JVM_GC_YOUNG_MEMORY_ALLOCATED_BYTES.toString(), MetricLevel.CORE); + + final AtomicLong firstYoungHeapPoolSizeAfterGc = new AtomicLong(); + // long live heap pool includes old gen heap pool and non-generation heap pool. + final AtomicLong longLivedHeapPoolSizeAfterGc = new AtomicLong(); + + // start watching for GC notifications for (GarbageCollectorMXBean mbean : ManagementFactory.getGarbageCollectorMXBeans()) { if (!(mbean instanceof NotificationEmitter)) { continue; @@ -194,16 +199,15 @@ public void bindTo(AbstractMetricService metricService) { previousTotal.set(total); } - String timerName; - if (isConcurrentPhase(gcCause, notificationInfo.getGcName())) { - timerName = "jvm_gc_concurrent_phase_time"; - } else { - timerName = "jvm_gc_pause"; - } // create a timer with tags named by gcCause, which binds gcCause with gcDuration Timer timer = metricService.getOrCreateTimer( - timerName, MetricLevel.CORE, "action", gcAction, "cause", gcCause); + SystemMetric.JVM_GC_PAUSE.toString(), + MetricLevel.CORE, + "action", + gcAction, + "cause", + gcCause); timer.update(duration, TimeUnit.MILLISECONDS); // add support for ZGC @@ -219,25 +223,6 @@ public void bindTo(AbstractMetricService metricService) { pausesCount.inc(); } - // monitoring old/young GC count, which is helpful for users to locate GC exception. - // Unfortunately, the JMX doesn't seem to provide an api for monitoring mixed gc in G1. - // In fact, JMX may treat mixed GCs as minor GCs. - if (GcGenerationAge.fromName(notificationInfo.getGcName()) == GcGenerationAge.OLD) { - Counter oldGcCounter = - metricService.getOrCreateCounter( - SystemMetric.JVM_GC_YOUNG_GC_COUNT.toString(), MetricLevel.CORE); - oldGcCounter.inc(); - } else if (GcGenerationAge.fromName(notificationInfo.getGcName()) - == GcGenerationAge.YOUNG) { - Counter youngGcCounter = - metricService.getOrCreateCounter( - SystemMetric.JVM_GC_OLD_GC_COUNT.toString(), MetricLevel.CORE); - youngGcCounter.inc(); - } - - // update memory usage percentage - heapMemUsedPercentage.set(calculateMemoryUsagePercentage()); - // Update promotion and allocation counters final Map before = gcInfo.getMemoryUsageBeforeGc(); final Map after = gcInfo.getMemoryUsageAfterGc(); @@ -246,8 +231,8 @@ public void bindTo(AbstractMetricService metricService) { countPoolSizeDelta( gcInfo.getMemoryUsageBeforeGc(), gcInfo.getMemoryUsageAfterGc(), - allocatedBytes, - heapPoolSizeAfterGc, + nonGenAllocatedBytes, + longLivedHeapPoolSizeAfterGc, nonGenerationalMemoryPool); if (after.get(nonGenerationalMemoryPool).getUsed() < before.get(nonGenerationalMemoryPool).getUsed()) { @@ -272,8 +257,7 @@ public void bindTo(AbstractMetricService metricService) { // GC (since in JMX, a minor GC of G1 may actually represent mixed GC, which collect // some obj in old gen region). To track the // live data size we record the value if we see a reduction in the old gen heap size - // or - // after a major GC. + // or after a major GC. if (oldAfter < oldBefore || GcGenerationAge.fromName(notificationInfo.getGcName()) == GcGenerationAge.OLD) { @@ -281,15 +265,21 @@ public void bindTo(AbstractMetricService metricService) { final long oldMaxAfter = after.get(oldGenPoolName).getMax(); maxDataSize.set(oldMaxAfter); } + countPoolSizeDelta( + gcInfo.getMemoryUsageBeforeGc(), + gcInfo.getMemoryUsageAfterGc(), + oldGenAllocatedBytes, + longLivedHeapPoolSizeAfterGc, + oldGenPoolName); } - if (youngGenPoolName != null) { + if (firstYoungGenPoolName != null) { countPoolSizeDelta( gcInfo.getMemoryUsageBeforeGc(), gcInfo.getMemoryUsageAfterGc(), - allocatedBytes, - heapPoolSizeAfterGc, - youngGenPoolName); + youngGenAllocatedBytes, + firstYoungHeapPoolSizeAfterGc, + firstYoungGenPoolName); } } }; @@ -321,12 +311,22 @@ public void unbindFrom(AbstractMetricService metricService) { metricService.remove(MetricType.AUTO_GAUGE, SystemMetric.JVM_GC_MAX_DATA_SIZE_BYTES.toString()); metricService.remove( MetricType.AUTO_GAUGE, SystemMetric.JVM_GC_LIVE_DATA_SIZE_BYTES.toString()); - metricService.remove(MetricType.COUNTER, SystemMetric.JVM_GC_MEMORY_ALLOCATED_BYTES.toString()); - metricService.remove(MetricType.AUTO_GAUGE, SystemMetric.JVM_GC_MEMORY_USED_PERCENT.toString()); - if (oldGenPoolName != null) { + if (nonGenerationalMemoryPool != null) { metricService.remove( - MetricType.COUNTER, SystemMetric.JVM_GC_MEMORY_PROMOTED_BYTES.toString()); + MetricType.COUNTER, SystemMetric.JVM_GC_NON_GEN_MEMORY_ALLOCATED_BYTES.toString()); + } else { + if (oldGenPoolName != null) { + metricService.remove( + MetricType.COUNTER, SystemMetric.JVM_GC_MEMORY_PROMOTED_BYTES.toString()); + metricService.remove( + MetricType.COUNTER, SystemMetric.JVM_GC_OLD_MEMORY_ALLOCATED_BYTES.toString()); + } + + if (firstYoungGenPoolName != null) { + metricService.remove( + MetricType.COUNTER, SystemMetric.JVM_GC_YOUNG_MEMORY_ALLOCATED_BYTES.toString()); + } } // start watching for GC notifications @@ -342,13 +342,14 @@ public void unbindFrom(AbstractMetricService metricService) { String gcCause = notificationInfo.getGcCause(); String gcAction = notificationInfo.getGcAction(); - String timerName; - if (isConcurrentPhase(gcCause, notificationInfo.getGcName())) { - timerName = "jvm_gc_concurrent_phase_time"; - } else { - timerName = "jvm_gc_pause"; - } - metricService.remove(MetricType.TIMER, timerName, "action", gcAction, "cause", gcCause); + metricService.remove( + MetricType.TIMER, + SystemMetric.JVM_GC_PAUSE.toString(), + "action", + gcAction, + "cause", + gcCause); + if (mbean.getName().equals("ZGC Cycles")) { metricService.remove( MetricType.COUNTER, SystemMetric.JVM_ZGC_CYCLES_COUNT.toString()); @@ -356,13 +357,6 @@ public void unbindFrom(AbstractMetricService metricService) { metricService.remove( MetricType.COUNTER, SystemMetric.JVM_ZGC_PAUSES_COUNT.toString()); } - if (GcGenerationAge.fromName(notificationInfo.getGcName()) == GcGenerationAge.OLD) { - metricService.remove( - MetricType.COUNTER, SystemMetric.JVM_GC_YOUNG_GC_COUNT.toString()); - } else if (GcGenerationAge.fromName(notificationInfo.getGcName()) - == GcGenerationAge.YOUNG) { - metricService.remove(MetricType.COUNTER, SystemMetric.JVM_GC_OLD_GC_COUNT.toString()); - } }; NotificationEmitter notificationEmitter = (NotificationEmitter) mbean; notificationEmitter.addNotificationListener( @@ -426,12 +420,6 @@ public void close() { notificationListenerCleanUpRunnables.forEach(Runnable::run); } - public long calculateMemoryUsagePercentage() { - return (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed() - * 100 - / Runtime.getRuntime().maxMemory()); - } - enum GcGenerationAge { OLD, YOUNG, diff --git a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java index 645c2307eeaa4..e04586c0f11bf 100644 --- a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java +++ b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/SystemMetric.java @@ -79,13 +79,13 @@ public enum SystemMetric { JVM_THREADS_STATUS_THREADS("jvm_threads_states_threads"), JVM_GC_MAX_DATA_SIZE_BYTES("jvm_gc_max_data_size_bytes"), JVM_GC_LIVE_DATA_SIZE_BYTES("jvm_gc_live_data_size_bytes"), - JVM_GC_MEMORY_ALLOCATED_BYTES("jvm_gc_memory_allocated_bytes"), + JVM_GC_YOUNG_MEMORY_ALLOCATED_BYTES("jvm_gc_young_memory_allocated_bytes"), + JVM_GC_OLD_MEMORY_ALLOCATED_BYTES("jvm_gc_old_memory_allocated_bytes"), + JVM_GC_NON_GEN_MEMORY_ALLOCATED_BYTES("jvm_gc_non_gen_memory_allocated_bytes"), JVM_GC_MEMORY_PROMOTED_BYTES("jvm_gc_memory_promoted_bytes"), - JVM_GC_MEMORY_USED_PERCENT("jvm_gc_memory_used_percent"), + JVM_GC_PAUSE("jvm_gc_pause"), JVM_ZGC_CYCLES_COUNT("jvm_zgc_cycles_count"), JVM_ZGC_PAUSES_COUNT("jvm_zgc_pauses_count"), - JVM_GC_YOUNG_GC_COUNT("jvm_gc_old_gc_count"), - JVM_GC_OLD_GC_COUNT("jvm_gc_young_gc_count"), JVM_GC_ACCUMULATED_TIME_PERCENTAGE("jvm_gc_accumulated_time_percentage"), // net related RECEIVED_BYTES("received_bytes"),