Skip to content

Add 'deserialize_resource' metric item in compaction #13298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jt2594838 merged 2 commits into from
Aug 27, 2024
Merged

Add 'deserialize_resource' metric item in compaction #13298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f74f8d2
add 'deserialize_resource' metric
shuwenwei Aug 26, 2024
ab6163c
fix tag
shuwenwei Aug 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
331 changes: 91 additions & 240 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion ...e/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl;

import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.AbstractCompactionWriter;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.RepairUnsortedFileCompactionWriter;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
Expand Down Expand Up @@ -66,7 +67,7 @@ private void prepareTargetFile() throws IOException {
if (timeIndex instanceof ArrayDeviceTimeIndex) {
targetFile.setTimeIndex(timeIndex);
} else {
targetFile.setTimeIndex(seqSourceFile.buildDeviceTimeIndex());
targetFile.setTimeIndex(CompactionUtils.buildDeviceTimeIndex(seqSourceFile));
}
if (seqSourceFile.modFileExists()) {
Files.createLink(
Expand Down
17 changes: 17 additions & 0 deletions ...rg/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,14 @@
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.service.metric.enums.Tag;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.SystemMetric;

Expand Down Expand Up @@ -363,4 +366,18 @@ public static boolean isDiskHasSpace(double redundancy) {
}
return true;
}

public static ArrayDeviceTimeIndex buildDeviceTimeIndex(TsFileResource resource)
throws IOException {
long resourceFileSize =
new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).length();
CompactionTaskManager.getInstance().getCompactionReadOperationRateLimiter().acquire(1);
while (resourceFileSize > 0) {
int readSize = (int) Math.min(resourceFileSize, Integer.MAX_VALUE);
CompactionTaskManager.getInstance().getCompactionReadRateLimiter().acquire(readSize);
resourceFileSize -= readSize;
}
CompactionMetrics.getInstance().recordDeserializeResourceInfo(resourceFileSize);
return resource.buildDeviceTimeIndex();
}
}
3 changes: 2 additions & 1 deletion ...rg/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.repair;

import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionChunkReader;
import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileReader;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType;
Expand Down Expand Up @@ -290,6 +291,6 @@ private static ArrayDeviceTimeIndex getDeviceTimeIndex(TsFileResource resource)
if (timeIndex instanceof ArrayDeviceTimeIndex) {
return (ArrayDeviceTimeIndex) timeIndex;
}
return resource.buildDeviceTimeIndex();
return CompactionUtils.buildDeviceTimeIndex(resource);
}
}
3 changes: 2 additions & 1 deletion ...b/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex;
Expand Down Expand Up @@ -161,7 +162,7 @@ private ArrayDeviceTimeIndex getDeviceTimeIndexFromCache(TsFileResource resource
}
ITimeIndex timeIndex = resource.getTimeIndex();
if (timeIndex instanceof FileTimeIndex) {
timeIndex = resource.buildDeviceTimeIndex();
timeIndex = CompactionUtils.buildDeviceTimeIndex(resource);
}
deviceTimeIndexCache.put(resource, (ArrayDeviceTimeIndex) timeIndex);
return (ArrayDeviceTimeIndex) timeIndex;
Expand Down
2 changes: 1 addition & 1 deletion ...apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ private FileDirtyInfo selectFileBaseOnDirtyData(TsFileResource resource)
ModificationFile modFile = resource.getModFile();
ITimeIndex timeIndex = resource.getTimeIndex();
if (timeIndex instanceof FileTimeIndex) {
timeIndex = resource.buildDeviceTimeIndex();
timeIndex = CompactionUtils.buildDeviceTimeIndex(resource);
}
Set<IDeviceID> deletedDevices = new HashSet<>();
boolean hasExpiredTooLong = false;
Expand Down
3 changes: 2 additions & 1 deletion .../iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils;

import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleContext;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
Expand Down Expand Up @@ -87,7 +88,7 @@ private void prepareDeviceInfos() throws IOException {
hasDetailedDeviceInfo = false;
return;
}
ArrayDeviceTimeIndex timeIndex = resource.buildDeviceTimeIndex();
ArrayDeviceTimeIndex timeIndex = CompactionUtils.buildDeviceTimeIndex(resource);
for (IDeviceID deviceId : timeIndex.getDevices()) {
deviceInfoMap.put(
deviceId,
Expand Down
3 changes: 2 additions & 1 deletion ...src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.iotdb.db.storageengine.dataregion.utils;

import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex;
Expand Down Expand Up @@ -379,7 +380,7 @@ public static boolean validateTsFileResourcesHasNoOverlap(List<TsFileResource> r
if (resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE) {
// if time index is not device time index, then deserialize it from resource file
try {
timeIndex = resource.buildDeviceTimeIndex();
timeIndex = CompactionUtils.buildDeviceTimeIndex(resource);
} catch (IOException e) {
// skip such files
continue;
Expand Down