Skip to content

Fixed the concurrency bug for delete-table and compaction that may cause skipped deletion #17736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shuwenwei merged 1 commit into from
May 22, 2026
+51 −1
Merged

Fixed the concurrency bug for delete-table and compaction that may cause skipped deletion #17736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion ...-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2556,7 +2556,7 @@
* lock if return true, the caller is responsible for unlocking all the already-acquiring lock
* in needToUnLockList
*/
private boolean tryGetFLushLock(

Check warning on line 2559 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

A "Brain Method" was detected. Refactor it to reduce at least one of the following metrics: LOC from 81 to 64, Complexity from 19 to 14, Nesting Level from 5 to 2, Number of Variables from 17 to 6. 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AZ5I8ugjGcavrrDWE400&open=AZ5I8ugjGcavrrDWE400&pullRequest=17736
long waitTimeInMs,
IDeviceID singleDeviceId,
Filter globalTimeFilter,
Expand Down Expand Up @@ -2995,7 +2995,7 @@
}
}

public void deleteByTable(RelationalDeleteDataNode node) throws IOException {

Check warning on line 2998 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

A "Brain Method" was detected. Refactor it to reduce at least one of the following metrics: LOC from 97 to 64, Complexity from 17 to 14, Nesting Level from 7 to 2, Number of Variables from 24 to 6. 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AZ5I8ugjGcavrrDWE401&open=AZ5I8ugjGcavrrDWE401&pullRequest=17736
if (node.getDatabaseName() != null && !node.getDatabaseName().equals(databaseName)) {
// not targeted on this database, return
return;
Expand Down Expand Up @@ -3386,7 +3386,7 @@
}
}

private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, ModEntry deletion)

Check warning on line 3389 in iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

A "Brain Method" was detected. Refactor it to reduce at least one of the following metrics: LOC from 149 to 64, Complexity from 29 to 14, Nesting Level from 5 to 2, Number of Variables from 26 to 6. 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AZ5I8ugjGcavrrDWE402&open=AZ5I8ugjGcavrrDWE402&pullRequest=17736
throws IOException {
Set<ModificationFile> involvedModificationFiles = new HashSet<>();
List<TsFileResource> deletedByMods = new ArrayList<>();
Expand Down Expand Up @@ -3479,7 +3479,11 @@
}
}
if (matchSize == devicesInFile.size()) {
deletedByFiles.add(sealedTsFile);
if (sealedTsFile.setStatus(TsFileResourceStatus.DELETED)) {
deletedByFiles.add(sealedTsFile);
} else {
deletedByMods.add(sealedTsFile);
}
}

if (logger.isDebugEnabled()) {
Expand Down
46 changes: 46 additions & 0 deletions ...e/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager;
Expand All @@ -56,6 +57,7 @@
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.constant.InnerSequenceCompactionSelector;
Expand All @@ -65,8 +67,13 @@
import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy;
import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate;
import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
Expand All @@ -80,6 +87,7 @@
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.reader.IPointReader;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.write.record.TSRecord;
Expand Down Expand Up @@ -2004,6 +2012,44 @@ public void testDeleteDataDirectlyUnseqWriteModsOrDeleteFiles()
Assert.assertFalse(tsFileResourceUnSeq.anyModFileExists());
}

@Test
public void testTableDeletionFallsBackToModsForCompactingFile() throws Exception {
RelationalInsertTabletNode insertTabletNode = genInsertTabletNode(10, 0);
dataRegion.insertTablet(insertTabletNode);
dataRegion.syncCloseAllWorkingTsFileProcessors();

TsFileResource sourceTsFile = dataRegion.getTsFileManager().getTsFileList(true).get(0);
assertTrue(sourceTsFile.transformStatus(TsFileResourceStatus.COMPACTION_CANDIDATE));
assertTrue(sourceTsFile.transformStatus(TsFileResourceStatus.COMPACTING));

TsFileResource targetTsFile =
new TsFileResource(new File(sourceTsFile.getTsFile().getParent(), "target.tsfile"));
CompactionUtils.prepareCompactionModFiles(
Collections.singletonList(targetTsFile), Collections.singletonList(sourceTsFile));
Assert.assertFalse(targetTsFile.anyModFileExists());

TableDeletionEntry deletionEntry =
new TableDeletionEntry(
new DeletionPredicate(StatementTestUtils.tableName(), new NOP()),
new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE));
dataRegion.deleteByTable(
new RelationalDeleteDataNode(
new PlanNodeId("table-delete"), deletionEntry, dataRegion.getDatabaseName()));

assertTrue(sourceTsFile.getTsFile().exists());
assertTrue(sourceTsFile.isCompacting());
assertTrue(sourceTsFile.anyModFileExists());
assertTrue(targetTsFile.anyModFileExists());
try {
targetTsFile.getModFileForWrite().close();
Assert.assertEquals(
Collections.singletonList(deletionEntry),
ModificationFile.readAllModifications(targetTsFile.getTsFile(), false));
} finally {
targetTsFile.removeModFile();
}
}

@Test
public void testFlushSpecifiedResource()
throws IllegalPathException, WriteProcessException, ExecutionException, InterruptedException {
Expand Down
Loading