From 0309be6d8329b176f172f57bc0eaf99d33a179a6 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 1 Nov 2023 10:14:56 +0800 Subject: [PATCH 1/2] Pipe: remove unnecessary waiting operation for memory alloc calculation --- .../db/pipe/resource/memory/PipeMemoryManager.java | 7 ------- .../dataregion/wal/utils/WALInsertNodeCache.java | 10 +++++++++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index 895d5890a03d4..97688e7b87dc4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -170,13 +170,6 @@ public synchronized PipeMemoryBlock tryAllocate(long sizeInBytes) { return new PipeMemoryBlock(sizeToAllocateInBytes); } - try { - this.wait(MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOGGER.warn("tryAllocate: interrupted while waiting for available memory", e); - } - sizeToAllocateInBytes = Math.max(sizeToAllocateInBytes * 2 / 3, MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java index 44e2c287622bf..3286c54640da5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.utils; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -56,6 +58,7 @@ public class WALInsertNodeCache { private static final Logger LOGGER = LoggerFactory.getLogger(WALInsertNodeCache.class); private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); + private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); // LRU cache, find Pair by WALEntryPosition private final PipeMemoryBlock allocatedMemoryBlock; @@ -69,7 +72,12 @@ public class WALInsertNodeCache { private WALInsertNodeCache(Integer dataRegionId) { allocatedMemoryBlock = - PipeResourceManager.memory().tryAllocate(2 * CONFIG.getWalFileSizeThresholdInByte()); + PipeResourceManager.memory() + .tryAllocate( + (long) + Math.min( + 2 * CONFIG.getWalFileSizeThresholdInByte(), + CONFIG.getAllocateMemoryForPipe() * 0.8 / 5)); isBatchLoadEnabled = allocatedMemoryBlock.getMemoryUsageInBytes() >= CONFIG.getWalFileSizeThresholdInByte(); lruCache = From 49e38af73748d130301c7c4f445d673f6fc9f959 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 1 Nov 2023 10:22:18 +0800 Subject: [PATCH 2/2] Update WALInsertNodeCache.java --- .../storageengine/dataregion/wal/utils/WALInsertNodeCache.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java index 3286c54640da5..da651cf4322bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.utils; -import org.apache.iotdb.commons.conf.CommonConfig; -import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -58,7 +56,6 @@ public class WALInsertNodeCache { private static final Logger LOGGER = LoggerFactory.getLogger(WALInsertNodeCache.class); private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); - private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); // LRU cache, find Pair by WALEntryPosition private final PipeMemoryBlock allocatedMemoryBlock;