From f74f8d29722c385af7586a9b841d8a60b8d30a74 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 26 Aug 2024 18:25:48 +0800 Subject: [PATCH 1/2] add 'deserialize_resource' metric --- .../db/service/metrics/CompactionMetrics.java | 306 +++++------------- ...RepairUnsortedFileCompactionPerformer.java | 3 +- .../execute/utils/CompactionUtils.java | 17 + .../repair/RepairDataFileScanUtil.java | 3 +- .../AbstractCompactionEstimator.java | 3 +- .../selector/impl/SettleSelectorImpl.java | 2 +- .../utils/TsFileResourceCandidate.java | 3 +- .../dataregion/utils/TsFileResourceUtils.java | 3 +- 8 files changed, 117 insertions(+), 223 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java index 1e1afdbad9651..a34cd26ba5927 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java @@ -179,6 +179,7 @@ public void recordWriteInfo( // region compaction read info private Counter totalCompactionReadInfoCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; + private Counter deserializeResourceCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; private void bindReadInfo(AbstractMetricService metricService) { for (CompactionType compactionType : CompactionType.values()) { @@ -211,6 +212,12 @@ private void bindReadInfo(AbstractMetricService metricService) { totalCompactionReadInfoCounter = metricService.getOrCreateCounter( Metric.DATA_READ.toString(), MetricLevel.IMPORTANT, Tag.NAME.toString(), "compaction"); + deserializeResourceCounter = + metricService.getOrCreateCounter( + Metric.DATA_READ.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + "deserialize_resource"); } private void unbindReadInfo(AbstractMetricService metricService) { @@ -239,6 +246,11 @@ private void unbindReadInfo(AbstractMetricService metricService) { } metricService.remove( MetricType.COUNTER, Metric.DATA_READ.toString(), Tag.NAME.toString(), "compaction"); + metricService.remove( + MetricType.COUNTER, + Metric.DATA_READ.toString(), + Tag.NAME.toString(), + "deserialize_resource"); } public void recordReadInfo( @@ -250,6 +262,10 @@ public void recordReadInfo( totalCompactionReadInfoCounter.inc(byteNum); } + public void recordDeserializeResourceInfo(long byteNum) { + deserializeResourceCounter.inc(byteNum); + } + // endregion // region compaction summary info @@ -536,114 +552,46 @@ private void unbindTaskInfo(AbstractMetricService metricService) { unSeqCompactionCostTimer = DoNothingMetricManager.DO_NOTHING_TIMER; crossCompactionCostTimer = DoNothingMetricManager.DO_NOTHING_TIMER; settleCompactionCostTimer = DoNothingMetricManager.DO_NOTHING_TIMER; - metricService.remove( - MetricType.TIMER, Metric.COST_TASK.toString(), Tag.NAME.toString(), "inner_seq_compaction"); - metricService.remove( - MetricType.TIMER, - Metric.COST_TASK.toString(), - Tag.NAME.toString(), - "inner_unseq_compaction"); - metricService.remove( - MetricType.TIMER, Metric.COST_TASK.toString(), Tag.NAME.toString(), "cross_compaction"); - metricService.remove( - MetricType.TIMER, Metric.COST_TASK.toString(), Tag.NAME.toString(), "insertion_compaction"); - metricService.remove( - MetricType.TIMER, Metric.COST_TASK.toString(), Tag.NAME.toString(), "settle_compaction"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_cross", - Tag.STATUS.toString(), - "waiting"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_insertion", - Tag.STATUS.toString(), - "waiting"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_inner_seq", - Tag.STATUS.toString(), - "waiting"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_inner_unseq", - Tag.STATUS.toString(), - "waiting"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_settle", - Tag.STATUS.toString(), - "waiting"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_cross", - Tag.STATUS.toString(), - "running"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_insertion", - Tag.STATUS.toString(), - "running"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_inner_seq", - Tag.STATUS.toString(), - "running"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_inner_unseq", - Tag.STATUS.toString(), - "running"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.QUEUE.toString(), - Tag.NAME.toString(), - "compaction_settle", - Tag.STATUS.toString(), - "running"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_COUNT.toString(), - Tag.NAME.toString(), - "inner_seq"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_COUNT.toString(), - Tag.NAME.toString(), - "inner_unseq"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_COUNT.toString(), - Tag.NAME.toString(), - "cross"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_COUNT.toString(), - Tag.NAME.toString(), - "insertion"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_COUNT.toString(), - Tag.NAME.toString(), - "settle"); + for (String taskType : + Arrays.asList( + "inner_seq_compaction", + "inner_unseq_compaction", + "cross_compaction", + "insertion_compaction", + "settle_compaction")) { + metricService.remove( + MetricType.TIMER, Metric.COST_TASK.toString(), Tag.NAME.toString(), taskType); + } + for (String taskType : + Arrays.asList( + "compaction_cross", + "compaction_insertion", + "compaction_inner_seq", + "compaction_inner_unseq", + "compaction_settle")) { + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.QUEUE.toString(), + Tag.NAME.toString(), + taskType, + Tag.STATUS.toString(), + "waiting"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.QUEUE.toString(), + Tag.NAME.toString(), + taskType, + Tag.STATUS.toString(), + "running"); + } + for (String taskType : + Arrays.asList("inner_seq", "inner_unseq", "cross", "insertion", "settle")) { + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.COMPACTION_TASK_COUNT.toString(), + Tag.NAME.toString(), + taskType); + } } // endregion @@ -738,48 +686,20 @@ private void bindCompactionTaskMemory(AbstractMetricService metricService) { } private void unbindCompactionTaskMemory(AbstractMetricService metricService) { - metricService.remove( - MetricType.HISTOGRAM, Metric.COMPACTION_TASK_MEMORY.toString(), Tag.NAME.toString(), "seq"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_MEMORY.toString(), - Tag.NAME.toString(), - "unseq"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_MEMORY.toString(), - Tag.NAME.toString(), - "cross"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_MEMORY.toString(), - Tag.NAME.toString(), - "settle"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_MEMORY_DISTRIBUTION.toString(), - Tag.NAME.toString(), - "total"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_MEMORY_DISTRIBUTION.toString(), - Tag.NAME.toString(), - "cross"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_MEMORY_DISTRIBUTION.toString(), - Tag.NAME.toString(), - "seq"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_MEMORY_DISTRIBUTION.toString(), - Tag.NAME.toString(), - "unseq"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.COMPACTION_TASK_MEMORY_DISTRIBUTION.toString(), - Tag.NAME.toString(), - "settle"); + for (String taskType : Arrays.asList("seq", "unseq", "cross", "settle")) { + metricService.remove( + MetricType.HISTOGRAM, + Metric.COMPACTION_TASK_MEMORY.toString(), + Tag.NAME.toString(), + taskType); + } + for (String taskType : Arrays.asList("seq", "unseq", "cross", "settle", "total")) { + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.COMPACTION_TASK_MEMORY_DISTRIBUTION.toString(), + Tag.NAME.toString(), + taskType); + } } // endregion @@ -954,73 +874,25 @@ private void bindCompactionTaskSelection(AbstractMetricService metricService) { } private void unbindCompactionTaskSelection(AbstractMetricService metricService) { - metricService.remove( - MetricType.GAUGE, Metric.COMPACTION_TASK_SELECTION.toString(), Tag.NAME.toString(), "seq"); - metricService.remove( - MetricType.GAUGE, - Metric.COMPACTION_TASK_SELECTION.toString(), - Tag.NAME.toString(), - "unseq"); - metricService.remove( - MetricType.GAUGE, - Metric.COMPACTION_TASK_SELECTION.toString(), - Tag.NAME.toString(), - "cross"); - metricService.remove( - MetricType.GAUGE, - Metric.COMPACTION_TASK_SELECTION.toString(), - Tag.NAME.toString(), - "insertion"); - metricService.remove( - MetricType.GAUGE, - Metric.COMPACTION_TASK_SELECTION.toString(), - Tag.NAME.toString(), - "settle"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTION_COST.toString(), - Tag.NAME.toString(), - "seq"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTION_COST.toString(), - Tag.NAME.toString(), - "unseq"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTION_COST.toString(), - Tag.NAME.toString(), - "cross"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTION_COST.toString(), - Tag.NAME.toString(), - "insertion"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTION_COST.toString(), - Tag.NAME.toString(), - "settle"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTED_FILE.toString(), - Tag.NAME.toString(), - "seq"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTED_FILE.toString(), - Tag.NAME.toString(), - "unseq"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTED_FILE.toString(), - Tag.NAME.toString(), - "cross"); - metricService.remove( - MetricType.HISTOGRAM, - Metric.COMPACTION_TASK_SELECTED_FILE.toString(), - Tag.NAME.toString(), - "settle"); + for (String taskType : Arrays.asList("seq", "unseq", "cross", "insertion", "settle")) { + metricService.remove( + MetricType.GAUGE, + Metric.COMPACTION_TASK_SELECTION.toString(), + Tag.NAME.toString(), + taskType); + metricService.remove( + MetricType.HISTOGRAM, + Metric.COMPACTION_TASK_SELECTION_COST.toString(), + Tag.NAME.toString(), + taskType); + } + for (String taskType : Arrays.asList("seq", "unseq", "cross", "settle")) { + metricService.remove( + MetricType.HISTOGRAM, + Metric.COMPACTION_TASK_SELECTED_FILE.toString(), + Tag.NAME.toString(), + taskType); + } } // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java index c9f61a02cbd1d..4269169a3c3f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.AbstractCompactionWriter; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.RepairUnsortedFileCompactionWriter; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -66,7 +67,7 @@ private void prepareTargetFile() throws IOException { if (timeIndex instanceof ArrayDeviceTimeIndex) { targetFile.setTimeIndex(timeIndex); } else { - targetFile.setTimeIndex(seqSourceFile.buildDeviceTimeIndex()); + targetFile.setTimeIndex(CompactionUtils.buildDeviceTimeIndex(seqSourceFile)); } if (seqSourceFile.modFileExists()) { Files.createLink( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index 3de7756142f94..8bed69c0d8744 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -23,11 +23,14 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.db.service.metrics.CompactionMetrics; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; +import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.Modification; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.SystemMetric; @@ -363,4 +366,18 @@ public static boolean isDiskHasSpace(double redundancy) { } return true; } + + public static ArrayDeviceTimeIndex buildDeviceTimeIndex(TsFileResource resource) + throws IOException { + long resourceFileSize = + new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).length(); + CompactionTaskManager.getInstance().getCompactionReadOperationRateLimiter().acquire(1); + while (resourceFileSize > 0) { + int readSize = (int) Math.min(resourceFileSize, Integer.MAX_VALUE); + CompactionTaskManager.getInstance().getCompactionReadRateLimiter().acquire(readSize); + resourceFileSize -= readSize; + } + CompactionMetrics.getInstance().recordDeserializeResourceInfo(resourceFileSize); + return resource.buildDeviceTimeIndex(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index e94bd9c78352a..9bbd219b6c9a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.repair; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionChunkReader; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileReader; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; @@ -290,6 +291,6 @@ private static ArrayDeviceTimeIndex getDeviceTimeIndex(TsFileResource resource) if (timeIndex instanceof ArrayDeviceTimeIndex) { return (ArrayDeviceTimeIndex) timeIndex; } - return resource.buildDeviceTimeIndex(); + return CompactionUtils.buildDeviceTimeIndex(resource); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java index 5b9269558d89e..37955f7cacbc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; @@ -161,7 +162,7 @@ private ArrayDeviceTimeIndex getDeviceTimeIndexFromCache(TsFileResource resource } ITimeIndex timeIndex = resource.getTimeIndex(); if (timeIndex instanceof FileTimeIndex) { - timeIndex = resource.buildDeviceTimeIndex(); + timeIndex = CompactionUtils.buildDeviceTimeIndex(resource); } deviceTimeIndexCache.put(resource, (ArrayDeviceTimeIndex) timeIndex); return (ArrayDeviceTimeIndex) timeIndex; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java index 7b0ab7a38a7f2..5b72740933bb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java @@ -202,7 +202,7 @@ private FileDirtyInfo selectFileBaseOnDirtyData(TsFileResource resource) ModificationFile modFile = resource.getModFile(); ITimeIndex timeIndex = resource.getTimeIndex(); if (timeIndex instanceof FileTimeIndex) { - timeIndex = resource.buildDeviceTimeIndex(); + timeIndex = CompactionUtils.buildDeviceTimeIndex(resource); } Set deletedDevices = new HashSet<>(); boolean hasExpiredTooLong = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java index 89874dbb4aba1..f01ed280ad5a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleContext; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -87,7 +88,7 @@ private void prepareDeviceInfos() throws IOException { hasDetailedDeviceInfo = false; return; } - ArrayDeviceTimeIndex timeIndex = resource.buildDeviceTimeIndex(); + ArrayDeviceTimeIndex timeIndex = CompactionUtils.buildDeviceTimeIndex(resource); for (IDeviceID deviceId : timeIndex.getDevices()) { deviceInfoMap.put( deviceId, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java index 36eff12124a91..0b0bc60be472a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.utils; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; @@ -379,7 +380,7 @@ public static boolean validateTsFileResourcesHasNoOverlap(List r if (resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE) { // if time index is not device time index, then deserialize it from resource file try { - timeIndex = resource.buildDeviceTimeIndex(); + timeIndex = CompactionUtils.buildDeviceTimeIndex(resource); } catch (IOException e) { // skip such files continue; From ab6163c0fd49439a5331d2b5f782bc5494bea1d3 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Mon, 26 Aug 2024 18:59:17 +0800 Subject: [PATCH 2/2] fix tag --- .../db/service/metrics/CompactionMetrics.java | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java index a34cd26ba5927..7a9531350bcd5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java @@ -93,8 +93,6 @@ private CompactionMetrics() { } } - private Counter totalCompactionWriteInfoCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; - private void bindWriteInfo(AbstractMetricService metricService) { for (CompactionType compactionType : CompactionType.values()) { writeCounters.put( @@ -123,14 +121,6 @@ private void bindWriteInfo(AbstractMetricService metricService) { METADATA) }); } - totalCompactionWriteInfoCounter = - metricService.getOrCreateCounter( - Metric.DATA_WRITTEN.toString(), - MetricLevel.IMPORTANT, - Tag.NAME.toString(), - "compaction", - Tag.TYPE.toString(), - "total"); } private void unbindWriteInfo(AbstractMetricService metricService) { @@ -157,13 +147,6 @@ private void unbindWriteInfo(AbstractMetricService metricService) { Tag.NAME.toString(), METADATA); } - metricService.remove( - MetricType.COUNTER, - Metric.DATA_WRITTEN.toString(), - Tag.NAME.toString(), - "compaction", - Tag.TYPE.toString(), - "total"); } public void recordWriteInfo( @@ -172,13 +155,11 @@ public void recordWriteInfo( if (counters != null) { counters[dataType.getValue()].inc(byteNum); } - totalCompactionWriteInfoCounter.inc(byteNum); } // endregion // region compaction read info - private Counter totalCompactionReadInfoCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; private Counter deserializeResourceCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; private void bindReadInfo(AbstractMetricService metricService) { @@ -209,13 +190,12 @@ private void bindReadInfo(AbstractMetricService metricService) { METADATA) }); } - totalCompactionReadInfoCounter = - metricService.getOrCreateCounter( - Metric.DATA_READ.toString(), MetricLevel.IMPORTANT, Tag.NAME.toString(), "compaction"); deserializeResourceCounter = metricService.getOrCreateCounter( Metric.DATA_READ.toString(), MetricLevel.IMPORTANT, + Tag.TYPE.toString(), + "total", Tag.NAME.toString(), "deserialize_resource"); } @@ -244,11 +224,11 @@ private void unbindReadInfo(AbstractMetricService metricService) { Tag.NAME.toString(), METADATA); } - metricService.remove( - MetricType.COUNTER, Metric.DATA_READ.toString(), Tag.NAME.toString(), "compaction"); metricService.remove( MetricType.COUNTER, Metric.DATA_READ.toString(), + Tag.TYPE.toString(), + "total", Tag.NAME.toString(), "deserialize_resource"); } @@ -259,7 +239,6 @@ public void recordReadInfo( if (counters != null) { counters[dataType.getValue()].inc(byteNum); } - totalCompactionReadInfoCounter.inc(byteNum); } public void recordDeserializeResourceInfo(long byteNum) {