[GR-53541] [GR-53407] OutOfMemoryError-related fixes.

PullRequest: graal/17649
oracle · May 13, 2024 · 6be3d60 · 6be3d60
2 parents 259b37b + 7922f87
commit 6be3d60
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 8 deletions.
diff --git a/...atevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/...atevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java
@@ -24,6 +24,7 @@
  */
 package com.oracle.svm.core.genscavenge;
 
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
 import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer;
 import static com.oracle.svm.core.snippets.KnownIntrinsics.readReturnAddress;
 
@@ -73,6 +74,7 @@
 import com.oracle.svm.core.heap.OutOfMemoryUtil;
 import com.oracle.svm.core.heap.PhysicalMemory;
 import com.oracle.svm.core.heap.ReferenceHandler;
+import com.oracle.svm.core.heap.ReferenceHandlerThread;
 import com.oracle.svm.core.heap.ReferenceMapIndex;
 import com.oracle.svm.core.heap.RestrictHeapAccess;
 import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner;
@@ -86,6 +88,7 @@
 import com.oracle.svm.core.snippets.ImplicitExceptions;
 import com.oracle.svm.core.stack.JavaStackWalk;
 import com.oracle.svm.core.stack.JavaStackWalker;
+import com.oracle.svm.core.stack.StackOverflowCheck;
 import com.oracle.svm.core.thread.JavaVMOperation;
 import com.oracle.svm.core.thread.NativeVMOperation;
 import com.oracle.svm.core.thread.NativeVMOperationData;
@@ -192,12 +195,19 @@ boolean collectWithoutAllocating(GCCause cause, boolean forceFullGC) {
         enqueueCollectOperation(data);
 
         boolean outOfMemory = data.getOutOfMemory();
-        if (outOfMemory && SerialGCOptions.IgnoreMaxHeapSizeWhileInVMOperation.getValue() && VMOperation.isInProgress()) {
+        if (outOfMemory && SerialGCOptions.IgnoreMaxHeapSizeWhileInVMOperation.getValue() && inVMInternalCode()) {
             outOfMemory = false;
         }
         return outOfMemory;
     }
 
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    private static boolean inVMInternalCode() {
+        return VMOperation.isInProgress() ||
+                        ReferenceHandlerThread.isReferenceHandlerThread() ||
+                        StackOverflowCheck.singleton().isYellowZoneAvailable();
+    }
+
     @Uninterruptible(reason = "Used as a transition between uninterruptible and interruptible code", calleeMustBe = false)
     private void enqueueCollectOperation(CollectionVMOperationData data) {
         collectOperation.enqueue(data);

diff --git a/.../com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java b/.../com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java
@@ -103,7 +103,8 @@ public Integer getValue(OptionValues values) {
     @Option(help = "Develop demographics of the object references visited. Serial GC only.", type = OptionType.Debug)//
     public static final HostedOptionKey<Boolean> GreyToBlackObjRefDemographics = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly);
 
-    @Option(help = "Ignore the maximum heap size while a VM operation is executed.", type = OptionType.Expert)//
+    /* Option should be renamed, see GR-53798. */
+    @Option(help = "Ignore the maximum heap size while in VM-internal code.", type = OptionType.Expert)//
     public static final HostedOptionKey<Boolean> IgnoreMaxHeapSizeWhileInVMOperation = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly);
 
     private SerialGCOptions() {

diff --git a/...rc/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/StackOverflowCheckImpl.java b/...rc/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/StackOverflowCheckImpl.java
@@ -24,6 +24,7 @@
  */
 package com.oracle.svm.core.graal.snippets;
 
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
 import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT;
 import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY;
 import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability;
@@ -210,6 +211,7 @@ private static void onYellowZoneMadeAvailable(int oldState, int newState) {
     }
 
     @Override
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public boolean isYellowZoneAvailable() {
         return yellowZoneStateTL.get() > STATE_YELLOW_ENABLED;
     }

diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceHandlerThread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceHandlerThread.java
@@ -24,13 +24,16 @@
  */
 package com.oracle.svm.core.heap;
 
+import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
+
 import org.graalvm.nativeimage.CurrentIsolate;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.IsolateThread;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
 import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
 import com.oracle.svm.core.thread.ThreadingSupportImpl;
@@ -56,6 +59,7 @@ public static void start() {
         }
     }
 
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public static boolean isReferenceHandlerThread() {
         if (isSupported()) {
             return CurrentIsolate.getCurrentThread() == singleton().isolateThread;

diff --git a/.../src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MultiThreadedMonitorSupport.java b/.../src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MultiThreadedMonitorSupport.java
@@ -40,6 +40,7 @@
 
 import org.graalvm.nativeimage.Platforms;
 
+import com.oracle.svm.core.NeverInline;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.WeakIdentityHashMap;
 import com.oracle.svm.core.annotate.Alias;
@@ -504,17 +505,22 @@ protected JavaMonitor getOrCreateMonitorFromMap(Object obj, boolean createIfNotE
             if (existingMonitor != null || !createIfNotExisting) {
                 return existingMonitor;
             }
-            long startTicks = JfrTicks.elapsedTicks();
-            JavaMonitor newMonitor = newMonitorLock();
-            JavaMonitor previousEntry = additionalMonitors.put(obj, newMonitor);
-            VMError.guarantee(previousEntry == null, "Replaced monitor in secondary storage map");
-            JavaMonitorInflateEvent.emit(obj, startTicks, cause);
-            return newMonitor;
+            return createMonitorAndAddToMap(obj, cause);
         } finally {
             additionalMonitorsLock.unlock();
         }
     }
 
+    @NeverInline("Prevent deadlocks in case of an OutOfMemoryError.")
+    private JavaMonitor createMonitorAndAddToMap(Object obj, MonitorInflationCause cause) {
+        long startTicks = JfrTicks.elapsedTicks();
+        JavaMonitor newMonitor = newMonitorLock();
+        JavaMonitor previousEntry = additionalMonitors.put(obj, newMonitor);
+        VMError.guarantee(previousEntry == null, "Replaced monitor in secondary storage map");
+        JavaMonitorInflateEvent.emit(obj, startTicks, cause);
+        return newMonitor;
+    }
+
     protected JavaMonitor newMonitorLock() {
         return new JavaMonitor();
     }