Skip to content

Compaction create target file with data dir selection strategy #12063

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
OneSizeFitsQuorum merged 6 commits into from
Apr 19, 2024
Merged

Compaction create target file with data dir selection strategy #12063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e82d2cb
compaction create target file with data dir selection strategy
shuwenwei Feb 21, 2024
87c4645
compaction create target file with data dir selection strategy
shuwenwei Feb 21, 2024
ac35094
Merge branch 'iotdb_master' into compactionCreateTargetFileWithDataDi…
shuwenwei Mar 25, 2024
f16e517
Merge branch 'iotdb_master' into compactionCreateTargetFileWithDataDi…
shuwenwei Apr 16, 2024
61d497b
Merge branch 'iotdb_master' into compactionCreateTargetFileWithDataDi…
shuwenwei Apr 17, 2024
3bd5cd1
Merge branch 'iotdb_master' into compactionCreateTargetFileWithDataDi…
shuwenwei Apr 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...e/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;

import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException;
Expand Down Expand Up @@ -148,7 +149,7 @@ public InnerSpaceCompactionTask(
createSummary();
}

protected void prepare() throws IOException {
protected void prepare() throws IOException, DiskSpaceInsufficientException {
targetTsFileResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(
selectedTsFileResourceList, sequence);
Expand Down
99 changes: 68 additions & 31 deletions ...va/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,36 @@ public static String generateNewTsFilePathWithMkdir(
int innerSpaceCompactionCount,
int crossSpaceCompactionCount)
throws DiskSpaceInsufficientException {
return generateNewTsFilePathWithMkdir(
sequence,
logicalStorageGroup,
virtualStorageGroup,
timePartitionId,
time,
version,
innerSpaceCompactionCount,
crossSpaceCompactionCount,
TsFileConstant.TSFILE_SUFFIX);
}

public static String generateNewTsFilePathWithMkdir(
boolean sequence,
String logicalStorageGroup,
String virtualStorageGroup,
long timePartitionId,
long time,
long version,
int innerSpaceCompactionCount,
int crossSpaceCompactionCount,
String customSuffix)
throws DiskSpaceInsufficientException {
String tsFileDir =
generateTsFileDir(sequence, logicalStorageGroup, virtualStorageGroup, timePartitionId);
fsFactory.getFile(tsFileDir).mkdirs();
return tsFileDir
+ File.separator
+ generateNewTsFileName(
time, version, innerSpaceCompactionCount, crossSpaceCompactionCount);
time, version, innerSpaceCompactionCount, crossSpaceCompactionCount, customSuffix);
}

public static String generateTsFileDir(
Expand All @@ -93,14 +116,24 @@ public static String generateTsFileDir(

public static String generateNewTsFileName(
long time, long version, int innerSpaceCompactionCount, int crossSpaceCompactionCount) {
return generateNewTsFileName(
time, version, innerSpaceCompactionCount, crossSpaceCompactionCount, TSFILE_SUFFIX);
}

public static String generateNewTsFileName(
long time,
long version,
int innerSpaceCompactionCount,
int crossSpaceCompactionCount,
String customSuffix) {
return time
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ version
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ innerSpaceCompactionCount
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ crossSpaceCompactionCount
+ TsFileConstant.TSFILE_SUFFIX;
+ customSuffix;
}

public static TsFileName getTsFileName(String fileName) throws IOException {
Expand Down Expand Up @@ -189,7 +222,7 @@ public static File increaseCrossCompactionCnt(File tsFile) throws IOException {
* @throws IOException
*/
public static List<TsFileResource> getCrossCompactionTargetFileResources(
List<TsFileResource> seqResources) throws IOException {
List<TsFileResource> seqResources) throws IOException, DiskSpaceInsufficientException {
List<TsFileResource> targetFileResources = new ArrayList<>();
for (TsFileResource resource : seqResources) {
TsFileName tsFileName = getTsFileName(resource.getTsFile().getName());
Expand All @@ -198,15 +231,16 @@ public static List<TsFileResource> getCrossCompactionTargetFileResources(
targetFileResources.add(
new TsFileResource(
new File(
resource.getTsFile().getParent(),
tsFileName.time
+ FILE_NAME_SEPARATOR
+ tsFileName.version
+ FILE_NAME_SEPARATOR
+ tsFileName.innerCompactionCnt
+ FILE_NAME_SEPARATOR
+ tsFileName.crossCompactionCnt
+ IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX),
generateNewTsFilePathWithMkdir(
resource.isSeq(),
resource.getDatabaseName(),
resource.getDataRegionId(),
resource.getTimePartition(),
tsFileName.time,
tsFileName.version,
tsFileName.innerCompactionCnt,
tsFileName.crossCompactionCnt,
IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)),
TsFileResourceStatus.COMPACTING));
}
return targetFileResources;
Expand All @@ -222,7 +256,8 @@ public static List<TsFileResource> getCrossCompactionTargetFileResources(
* @throws IOException
*/
public static TsFileResource getInnerCompactionTargetFileResource(
List<TsFileResource> tsFileResources, boolean sequence) throws IOException {
List<TsFileResource> tsFileResources, boolean sequence)
throws IOException, DiskSpaceInsufficientException {
long minTime = Long.MAX_VALUE;
long maxTime = Long.MIN_VALUE;
long minVersion = Long.MAX_VALUE;
Expand All @@ -243,27 +278,29 @@ public static TsFileResource getInnerCompactionTargetFileResource(
sequence
? new TsFileResource(
new File(
tsFileResources.get(0).getTsFile().getParent(),
minTime
+ FILE_NAME_SEPARATOR
+ minVersion
+ FILE_NAME_SEPARATOR
+ (maxInnerMergeCount + 1)
+ FILE_NAME_SEPARATOR
+ maxCrossMergeCount
+ IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
generateNewTsFilePathWithMkdir(
sequence,
tsFileResources.get(0).getDatabaseName(),
tsFileResources.get(0).getDataRegionId(),
tsFileResources.get(0).getTimePartition(),
minTime,
minVersion,
(int) maxInnerMergeCount + 1,
(int) maxCrossMergeCount,
IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)),
TsFileResourceStatus.COMPACTING)
: new TsFileResource(
new File(
tsFileResources.get(0).getTsFile().getParent(),
maxTime
+ FILE_NAME_SEPARATOR
+ maxVersion
+ FILE_NAME_SEPARATOR
+ (maxInnerMergeCount + 1)
+ FILE_NAME_SEPARATOR
+ maxCrossMergeCount
+ IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
generateNewTsFilePathWithMkdir(
sequence,
tsFileResources.get(0).getDatabaseName(),
tsFileResources.get(0).getDataRegionId(),
tsFileResources.get(0).getTimePartition(),
maxTime,
maxVersion,
(int) maxInnerMergeCount + 1,
(int) maxCrossMergeCount,
IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)),
TsFileResourceStatus.COMPACTING);
resource.setSeq(sequence);
return resource;
Expand Down
9 changes: 6 additions & 3 deletions ...db/storageengine/dataregion/compaction/inner/InnerSeqCompactionWithFastPerformerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ public void testDeserializePage() throws MetadataException, IOException, WritePr
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1);
CompactionFileGeneratorUtils.generateTsFileResource(
true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
sourceResources.add(tsFileResource);
Expand Down Expand Up @@ -466,7 +467,8 @@ public void testAppendPage() throws Exception {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1);
CompactionFileGeneratorUtils.generateTsFileResource(
true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
Expand Down Expand Up @@ -791,7 +793,8 @@ public void testAppendChunk() throws Exception {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1);
CompactionFileGeneratorUtils.generateTsFileResource(
true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
Expand Down
9 changes: 6 additions & 3 deletions ...orageengine/dataregion/compaction/inner/InnerSeqCompactionWithReadChunkPerformerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ public void testDeserializePage() throws Exception {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1);
CompactionFileGeneratorUtils.generateTsFileResource(
true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
sourceResources.add(tsFileResource);
Expand Down Expand Up @@ -459,7 +460,8 @@ public void testAppendPage() throws Exception {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1);
CompactionFileGeneratorUtils.generateTsFileResource(
true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
Expand Down Expand Up @@ -780,7 +782,8 @@ public void testAppendChunk() throws Exception {
pagePointsNum.add(300L);
chunkPagePointsNum.add(pagePointsNum);
TsFileResource tsFileResource =
CompactionFileGeneratorUtils.generateTsFileResource(true, i + 1);
CompactionFileGeneratorUtils.generateTsFileResource(
true, i + 1, COMPACTION_TEST_SG);
CompactionFileGeneratorUtils.writeTsFile(
fullPath, chunkPagePointsNum, i * 600L, tsFileResource);
toMergeResources.add(tsFileResource);
Expand Down
10 changes: 8 additions & 2 deletions ...ache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionFileGeneratorUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
Expand Down Expand Up @@ -69,14 +70,15 @@ public static TsFileResource getTargetTsFileResourceFromSourceResource(
}

public static List<TsFileResource> getInnerCompactionTargetTsFileResources(
List<TsFileResource> fileResources, boolean seq) throws IOException {
List<TsFileResource> fileResources, boolean seq)
throws IOException, DiskSpaceInsufficientException {
List<TsFileResource> resources = new ArrayList<>();
resources.add(TsFileNameGenerator.getInnerCompactionTargetFileResource(fileResources, seq));
return resources;
}

public static List<TsFileResource> getCrossCompactionTargetTsFileResources(
List<TsFileResource> seqFileResources) throws IOException {
List<TsFileResource> seqFileResources) throws IOException, DiskSpaceInsufficientException {
return TsFileNameGenerator.getCrossCompactionTargetFileResources(seqFileResources);
}

Expand All @@ -90,6 +92,8 @@ public static TsFileResource generateTsFileResource(
return new TsFileResource(
new File(
TestConstant.BASE_OUTPUT_PATH
.concat(File.separator)
.concat("data")
.concat(File.separator)
.concat("sequence")
.concat(File.separator)
Expand All @@ -112,6 +116,8 @@ public static TsFileResource generateTsFileResource(
return new TsFileResource(
new File(
TestConstant.BASE_OUTPUT_PATH
.concat(File.separator)
.concat("data")
.concat(File.separator)
.concat("unsequence")
.concat(File.separator)
Expand Down