From 7d53f1a89e95e91b3a69f093b91d7fc18841d393 Mon Sep 17 00:00:00 2001 From: HTHou Date: Thu, 28 Nov 2024 16:56:59 +0800 Subject: [PATCH 1/3] Fix async recover TsfileResource error handle --- .../iotdb/db/storageengine/dataregion/DataRegion.java | 3 ++- .../dataregion/utils/TsFileResourceUtils.java | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index a07aeafc6a4a0..4721d27eff71c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -888,7 +888,8 @@ private Callable recoverFilesInPartition( Callable asyncRecoverTask = null; for (TsFileResource tsFileResource : resourceList) { tsFileManager.add(tsFileResource, isSeq); - if (fileTimeIndexMap.containsKey(tsFileResource.getTsFileID())) { + if (fileTimeIndexMap.containsKey(tsFileResource.getTsFileID()) + && tsFileResource.resourceFileExists()) { tsFileResource.setTimeIndex(fileTimeIndexMap.get(tsFileResource.getTsFileID())); tsFileResource.setStatus(TsFileResourceStatus.NORMAL); resourceListForAsyncRecover.add(tsFileResource); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java index 4ab7f5f109b6c..6493cd9ca34d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.utils; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; @@ -61,6 +63,7 @@ public class TsFileResourceUtils { private static final Logger logger = LoggerFactory.getLogger(TsFileResourceUtils.class); + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final String VALIDATE_FAILED = "validate failed,"; private TsFileResourceUtils() { @@ -425,14 +428,15 @@ public static void updateTsFileResource( public static void updateTsFileResource( Map> device2Metadata, TsFileResource tsFileResource) { + ITimeIndex newTimeIndex = config.getTimeIndexLevel().getTimeIndex(); for (Map.Entry> entry : device2Metadata.entrySet()) { for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) { - tsFileResource.updateStartTime( + newTimeIndex.updateStartTime( entry.getKey(), timeseriesMetaData.getStatistics().getStartTime()); - tsFileResource.updateEndTime( - entry.getKey(), timeseriesMetaData.getStatistics().getEndTime()); + newTimeIndex.updateEndTime(entry.getKey(), timeseriesMetaData.getStatistics().getEndTime()); } } + tsFileResource.setTimeIndex(newTimeIndex); } /** From f5964392c0a8043097d48a93a3b51dd84b1f340a Mon Sep 17 00:00:00 2001 From: HTHou Date: Thu, 28 Nov 2024 17:27:13 +0800 Subject: [PATCH 2/3] fix compile --- .../db/storageengine/dataregion/utils/TsFileResourceUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java index 6493cd9ca34d9..93d2b11d1a0e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; From 64ed830d97b0df43177863bfc8224a76c2757d75 Mon Sep 17 00:00:00 2001 From: HTHou Date: Thu, 28 Nov 2024 18:43:46 +0800 Subject: [PATCH 3/3] fix IT --- .../storageengine/dataregion/utils/TsFileResourceUtils.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java index 93d2b11d1a0e4..05ee0d256a1ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java @@ -429,7 +429,11 @@ public static void updateTsFileResource( public static void updateTsFileResource( Map> device2Metadata, TsFileResource tsFileResource) { - ITimeIndex newTimeIndex = config.getTimeIndexLevel().getTimeIndex(); + // For async recover tsfile, there might be a FileTimeIndex, we need a new newTimeIndex + ITimeIndex newTimeIndex = + tsFileResource.getTimeIndex().getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE + ? config.getTimeIndexLevel().getTimeIndex() + : tsFileResource.getTimeIndex(); for (Map.Entry> entry : device2Metadata.entrySet()) { for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) { newTimeIndex.updateStartTime(