Skip to content

Fix concurrency problem in Memory Control #12984

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
OneSizeFitsQuorum merged 3 commits into from
Jul 22, 2024
Merged

Fix concurrency problem in Memory Control #12984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a4a0e29
fix memory concurrency problem
SpriCoder Jul 22, 2024
8302be7
Merge branch 'master' into feature/optimize_memory
SpriCoder Jul 22, 2024
64e93d3
spotless
OneSizeFitsQuorum Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .../main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,7 @@ public synchronized void init() throws MetadataException {

if (config.getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS)) {
long memCost = config.getSchemaRatisConsensusLogAppenderBufferSizeMax();
if (!SystemInfo.getInstance()
.addDirectBufferMemoryCost(config.getSchemaRatisConsensusLogAppenderBufferSizeMax())) {
if (!SystemInfo.getInstance().addDirectBufferMemoryCost(memCost)) {
throw new MetadataException(
"Total allocated memory for direct buffer will be "
+ (SystemInfo.getInstance().getDirectBufferMemoryCost() + memCost)
Expand Down
3 changes: 1 addition & 2 deletions .../main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,7 @@ public synchronized void init() throws MetadataException {

if (config.getSchemaRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS)) {
long memCost = config.getSchemaRatisConsensusLogAppenderBufferSizeMax();
if (!SystemInfo.getInstance()
.addDirectBufferMemoryCost(config.getSchemaRatisConsensusLogAppenderBufferSizeMax())) {
if (!SystemInfo.getInstance().addDirectBufferMemoryCost(memCost)) {
throw new MetadataException(
"Total allocated memory for direct buffer will be "
+ (SystemInfo.getInstance().getDirectBufferMemoryCost() + memCost)
Expand Down
20 changes: 11 additions & 9 deletions ...re/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.util.PriorityQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

Expand Down Expand Up @@ -196,15 +197,16 @@ public synchronized void resetFlushingMemTableCost(long flushingMemTableCost) {
}

public boolean addDirectBufferMemoryCost(long size) {
while (true) {
long memCost = directBufferMemoryCost.get();
if (memCost + size > totalDirectBufferMemorySizeLimit) {
return false;
}
if (directBufferMemoryCost.compareAndSet(memCost, memCost + size)) {
return true;
}
}
AtomicBoolean result = new AtomicBoolean(false);
directBufferMemoryCost.updateAndGet(
memCost -> {
if (memCost + size > totalDirectBufferMemorySizeLimit) {
return memCost;
}
result.set(true);
return memCost + size;
});
return result.get();
}

public void decreaseDirectBufferMemoryCost(long size) {
Expand Down