From e82d2cb09e29e65923f6d9903806d43e006bb7d0 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 21 Feb 2024 15:40:38 +0800 Subject: [PATCH 1/2] compaction create target file with data dir selection strategy --- .../task/InnerSpaceCompactionTask.java | 3 +- .../tsfile/generator/TsFileNameGenerator.java | 98 +++++++++++++------ ...nerSeqCompactionWithFastPerformerTest.java | 9 +- ...qCompactionWithReadChunkPerformerTest.java | 9 +- .../utils/CompactionFileGeneratorUtils.java | 10 +- 5 files changed, 91 insertions(+), 38 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java index 5219ab6bbf199..9a1aadf7fb935 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; import org.apache.iotdb.db.service.metrics.CompactionMetrics; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException; @@ -147,7 +148,7 @@ public InnerSpaceCompactionTask( createSummary(); } - protected void prepare() throws IOException { + protected void prepare() throws IOException, DiskSpaceInsufficientException { targetTsFileResource = TsFileNameGenerator.getInnerCompactionTargetFileResource( selectedTsFileResourceList, sequence); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java index d9a2ce07c7ae6..203afef7e4a48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java @@ -74,6 +74,26 @@ public static String generateNewTsFilePathWithMkdir( time, version, innerSpaceCompactionCount, crossSpaceCompactionCount); } + public static String generateNewTsFilePathWithMkdir( + boolean sequence, + String logicalStorageGroup, + String virtualStorageGroup, + long timePartitionId, + long time, + long version, + int innerSpaceCompactionCount, + int crossSpaceCompactionCount, + String customSuffix) + throws DiskSpaceInsufficientException { + String tsFileDir = + generateTsFileDir(sequence, logicalStorageGroup, virtualStorageGroup, timePartitionId); + fsFactory.getFile(tsFileDir).mkdirs(); + return tsFileDir + + File.separator + + generateNewTsFileName( + time, version, innerSpaceCompactionCount, crossSpaceCompactionCount, customSuffix); + } + public static String generateTsFileDir( boolean sequence, String logicalStorageGroup, @@ -103,6 +123,22 @@ public static String generateNewTsFileName( + TsFileConstant.TSFILE_SUFFIX; } + public static String generateNewTsFileName( + long time, + long version, + int innerSpaceCompactionCount, + int crossSpaceCompactionCount, + String customSuffix) { + return time + + IoTDBConstant.FILE_NAME_SEPARATOR + + version + + IoTDBConstant.FILE_NAME_SEPARATOR + + innerSpaceCompactionCount + + IoTDBConstant.FILE_NAME_SEPARATOR + + crossSpaceCompactionCount + + customSuffix; + } + public static TsFileName getTsFileName(String fileName) throws IOException { Matcher matcher = TsFileName.FILE_NAME_MATCHER.matcher(fileName); if (matcher.find()) { @@ -189,7 +225,7 @@ public static File increaseCrossCompactionCnt(File tsFile) throws IOException { * @throws IOException */ public static List getCrossCompactionTargetFileResources( - List seqResources) throws IOException { + List seqResources) throws IOException, DiskSpaceInsufficientException { List targetFileResources = new ArrayList<>(); for (TsFileResource resource : seqResources) { TsFileName tsFileName = getTsFileName(resource.getTsFile().getName()); @@ -198,15 +234,16 @@ public static List getCrossCompactionTargetFileResources( targetFileResources.add( new TsFileResource( new File( - resource.getTsFile().getParent(), - tsFileName.time - + FILE_NAME_SEPARATOR - + tsFileName.version - + FILE_NAME_SEPARATOR - + tsFileName.innerCompactionCnt - + FILE_NAME_SEPARATOR - + tsFileName.crossCompactionCnt - + IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX), + generateNewTsFilePathWithMkdir( + resource.isSeq(), + resource.getDatabaseName(), + resource.getDataRegionId(), + resource.getTimePartition(), + tsFileName.time, + tsFileName.version, + tsFileName.innerCompactionCnt, + tsFileName.crossCompactionCnt, + IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)), TsFileResourceStatus.COMPACTING)); } return targetFileResources; @@ -222,7 +259,8 @@ public static List getCrossCompactionTargetFileResources( * @throws IOException */ public static TsFileResource getInnerCompactionTargetFileResource( - List tsFileResources, boolean sequence) throws IOException { + List tsFileResources, boolean sequence) + throws IOException, DiskSpaceInsufficientException { long minTime = Long.MAX_VALUE; long maxTime = Long.MIN_VALUE; long minVersion = Long.MAX_VALUE; @@ -243,27 +281,29 @@ public static TsFileResource getInnerCompactionTargetFileResource( sequence ? new TsFileResource( new File( - tsFileResources.get(0).getTsFile().getParent(), - minTime - + FILE_NAME_SEPARATOR - + minVersion - + FILE_NAME_SEPARATOR - + (maxInnerMergeCount + 1) - + FILE_NAME_SEPARATOR - + maxCrossMergeCount - + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX), + generateNewTsFilePathWithMkdir( + sequence, + tsFileResources.get(0).getDatabaseName(), + tsFileResources.get(0).getDataRegionId(), + tsFileResources.get(0).getTimePartition(), + minTime, + minVersion, + (int) maxInnerMergeCount + 1, + (int) maxCrossMergeCount, + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)), TsFileResourceStatus.COMPACTING) : new TsFileResource( new File( - tsFileResources.get(0).getTsFile().getParent(), - maxTime - + FILE_NAME_SEPARATOR - + maxVersion - + FILE_NAME_SEPARATOR - + (maxInnerMergeCount + 1) - + FILE_NAME_SEPARATOR - + maxCrossMergeCount - + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX), + generateNewTsFilePathWithMkdir( + sequence, + tsFileResources.get(0).getDatabaseName(), + tsFileResources.get(0).getDataRegionId(), + tsFileResources.get(0).getTimePartition(), + maxTime, + maxVersion, + (int) maxInnerMergeCount + 1, + (int) maxCrossMergeCount, + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)), TsFileResourceStatus.COMPACTING); resource.setSeq(sequence); return resource; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java index b8f7f7151bf53..539da5cc4eb19 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java @@ -171,7 +171,8 @@ public void testDeserializePage() throws MetadataException, IOException, WritePr pagePointsNum.add(300L); chunkPagePointsNum.add(pagePointsNum); TsFileResource tsFileResource = - CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1); + CompactionFileGeneratorUtils.generateTsFileResource( + true, i + 1, COMPACTION_TEST_SG); CompactionFileGeneratorUtils.writeTsFile( fullPath, chunkPagePointsNum, i * 600L, tsFileResource); sourceResources.add(tsFileResource); @@ -466,7 +467,8 @@ public void testAppendPage() throws Exception { pagePointsNum.add(300L); chunkPagePointsNum.add(pagePointsNum); TsFileResource tsFileResource = - CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1); + CompactionFileGeneratorUtils.generateTsFileResource( + true, i + 1, COMPACTION_TEST_SG); CompactionFileGeneratorUtils.writeTsFile( fullPath, chunkPagePointsNum, i * 600L, tsFileResource); toMergeResources.add(tsFileResource); @@ -791,7 +793,8 @@ public void testAppendChunk() throws Exception { pagePointsNum.add(300L); chunkPagePointsNum.add(pagePointsNum); TsFileResource tsFileResource = - CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1); + CompactionFileGeneratorUtils.generateTsFileResource( + true, i + 1, COMPACTION_TEST_SG); CompactionFileGeneratorUtils.writeTsFile( fullPath, chunkPagePointsNum, i * 600L, tsFileResource); toMergeResources.add(tsFileResource); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java index f9ce303ecb995..aab06143e7b84 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java @@ -169,7 +169,8 @@ public void testDeserializePage() throws Exception { pagePointsNum.add(300L); chunkPagePointsNum.add(pagePointsNum); TsFileResource tsFileResource = - CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1); + CompactionFileGeneratorUtils.generateTsFileResource( + true, i + 1, COMPACTION_TEST_SG); CompactionFileGeneratorUtils.writeTsFile( fullPath, chunkPagePointsNum, i * 600L, tsFileResource); sourceResources.add(tsFileResource); @@ -459,7 +460,8 @@ public void testAppendPage() throws Exception { pagePointsNum.add(300L); chunkPagePointsNum.add(pagePointsNum); TsFileResource tsFileResource = - CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1); + CompactionFileGeneratorUtils.generateTsFileResource( + true, i + 1, COMPACTION_TEST_SG); CompactionFileGeneratorUtils.writeTsFile( fullPath, chunkPagePointsNum, i * 600L, tsFileResource); toMergeResources.add(tsFileResource); @@ -780,7 +782,8 @@ public void testAppendChunk() throws Exception { pagePointsNum.add(300L); chunkPagePointsNum.add(pagePointsNum); TsFileResource tsFileResource = - CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1); + CompactionFileGeneratorUtils.generateTsFileResource( + true, i + 1, COMPACTION_TEST_SG); CompactionFileGeneratorUtils.writeTsFile( fullPath, chunkPagePointsNum, i * 600L, tsFileResource); toMergeResources.add(tsFileResource); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java index 12a6dc73e2ada..00f9491fdfc42 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -67,14 +68,15 @@ public static TsFileResource getTargetTsFileResourceFromSourceResource( } public static List getInnerCompactionTargetTsFileResources( - List fileResources, boolean seq) throws IOException { + List fileResources, boolean seq) + throws IOException, DiskSpaceInsufficientException { List resources = new ArrayList<>(); resources.add(TsFileNameGenerator.getInnerCompactionTargetFileResource(fileResources, seq)); return resources; } public static List getCrossCompactionTargetTsFileResources( - List seqFileResources) throws IOException { + List seqFileResources) throws IOException, DiskSpaceInsufficientException { return TsFileNameGenerator.getCrossCompactionTargetFileResources(seqFileResources); } @@ -88,6 +90,8 @@ public static TsFileResource generateTsFileResource( return new TsFileResource( new File( TestConstant.BASE_OUTPUT_PATH + .concat(File.separator) + .concat("data") .concat(File.separator) .concat("sequence") .concat(File.separator) @@ -110,6 +114,8 @@ public static TsFileResource generateTsFileResource( return new TsFileResource( new File( TestConstant.BASE_OUTPUT_PATH + .concat(File.separator) + .concat("data") .concat(File.separator) .concat("unsequence") .concat(File.separator) From 87c46456dc71e5b6d472ca10086f112a12863513 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Wed, 21 Feb 2024 15:45:46 +0800 Subject: [PATCH 2/2] compaction create target file with data dir selection strategy --- .../tsfile/generator/TsFileNameGenerator.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java index 203afef7e4a48..040f93927b9da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java @@ -65,13 +65,16 @@ public static String generateNewTsFilePathWithMkdir( int innerSpaceCompactionCount, int crossSpaceCompactionCount) throws DiskSpaceInsufficientException { - String tsFileDir = - generateTsFileDir(sequence, logicalStorageGroup, virtualStorageGroup, timePartitionId); - fsFactory.getFile(tsFileDir).mkdirs(); - return tsFileDir - + File.separator - + generateNewTsFileName( - time, version, innerSpaceCompactionCount, crossSpaceCompactionCount); + return generateNewTsFilePathWithMkdir( + sequence, + logicalStorageGroup, + virtualStorageGroup, + timePartitionId, + time, + version, + innerSpaceCompactionCount, + crossSpaceCompactionCount, + TsFileConstant.TSFILE_SUFFIX); } public static String generateNewTsFilePathWithMkdir( @@ -113,14 +116,8 @@ public static String generateTsFileDir( public static String generateNewTsFileName( long time, long version, int innerSpaceCompactionCount, int crossSpaceCompactionCount) { - return time - + IoTDBConstant.FILE_NAME_SEPARATOR - + version - + IoTDBConstant.FILE_NAME_SEPARATOR - + innerSpaceCompactionCount - + IoTDBConstant.FILE_NAME_SEPARATOR - + crossSpaceCompactionCount - + TsFileConstant.TSFILE_SUFFIX; + return generateNewTsFileName( + time, version, innerSpaceCompactionCount, crossSpaceCompactionCount, TSFILE_SUFFIX); } public static String generateNewTsFileName(