8317273: compiler/codecache/OverflowCodeCacheTest.java fails transiently on Graal

src/hotspot/share/code/codeBlob.cpp

@@ -414,14 +414,21 @@ RuntimeStub* RuntimeStub::new_runtime_stub(const char* stub_name,
                                            int frame_complete,
                                            int frame_size,
                                            OopMapSet* oop_maps,
-                                           bool caller_must_gc_arguments)
+                                           bool caller_must_gc_arguments,
+                                           bool alloc_fail_is_fatal)
 {
   RuntimeStub* stub = nullptr;
   unsigned int size = CodeBlob::allocation_size(cb, sizeof(RuntimeStub));
   ThreadInVMfromUnknown __tiv;  // get to VM state in case we block on CodeCache_lock
   {
     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
     stub = new (size) RuntimeStub(stub_name, cb, size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments);
+    if (stub == nullptr) {
+      if (!alloc_fail_is_fatal) {
+        return nullptr;
+      }
+      fatal("Initial size of CodeCache is too small");
+    }
   }
 
   trace_new_stub(stub, "RuntimeStub - ", stub_name);
@@ -431,9 +438,7 @@ RuntimeStub* RuntimeStub::new_runtime_stub(const char* stub_name,
 
 
 void* RuntimeStub::operator new(size_t s, unsigned size) throw() {
-  void* p = CodeCache::allocate(size, CodeBlobType::NonNMethod);
-  if (!p) fatal("Initial size of CodeCache is too small");
-  return p;
+  return CodeCache::allocate(size, CodeBlobType::NonNMethod);
 }
 
 // operator new shared by all singletons:

src/hotspot/share/code/codeBlob.hpp

@@ -500,7 +500,8 @@ class RuntimeStub: public RuntimeBlob {
     int         frame_complete,
     int         frame_size,
     OopMapSet*  oop_maps,
-    bool        caller_must_gc_arguments
+    bool        caller_must_gc_arguments,
+    bool        alloc_fail_is_fatal=true
   );
 
   static void free(RuntimeStub* stub) { RuntimeBlob::free(stub); }

src/hotspot/share/jvmci/jvmciCodeInstaller.cpp

@@ -36,6 +36,7 @@
 #include "oops/klass.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/methodHandles.hpp"
+#include "runtime/arguments.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/jniHandles.inline.hpp"
 #include "runtime/os.hpp"
@@ -650,6 +651,53 @@ void CodeInstaller::initialize_dependencies(HotSpotCompiledCodeStream* stream, u
   }
 }
 
+JVMCI::CodeInstallResult CodeInstaller::install_runtime_stub(CodeBlob*& cb,
+                                                             const char* name,
+                                                             CodeBuffer* buffer,
+                                                             int stack_slots,
+                                                             JVMCI_TRAPS) {
+  if (name == nullptr) {
+    JVMCI_ERROR_OK("stub should have a name");
+  }
+
+  name = os::strdup(name);
+  GrowableArray<RuntimeStub*> *stubs_to_free = nullptr;
+#ifdef ASSERT
+  const char* val = Arguments::PropertyList_get_value(Arguments::system_properties(), "test.jvmci.forceRuntimeStubAllocFail");
+  if (val != nullptr && strstr(name , val) != 0) {
+    stubs_to_free = new GrowableArray<RuntimeStub*>();
+    JVMCI_event_1("forcing allocation of %s in code cache to fail", name);
+  }
+#endif
+
+  do {
+    RuntimeStub* stub = RuntimeStub::new_runtime_stub(name,
+                                       buffer,
+                                       _offsets.value(CodeOffsets::Frame_Complete),
+                                       stack_slots,
+                                       _debug_recorder->_oopmaps,
+                                       /* caller_must_gc_arguments */ false,
+                                       /* alloc_fail_is_fatal */ false);
+    cb = stub;
+    if (stub == nullptr) {
+      // Allocation failed
+#ifdef ASSERT
+      if (stubs_to_free != nullptr) {
+        JVMCI_event_1("allocation of %s in code cache failed, freeing %d stubs", name, stubs_to_free->length());
+        for (GrowableArrayIterator<RuntimeStub*> iter = stubs_to_free->begin(); iter != stubs_to_free->end(); ++iter) {
+          RuntimeStub::free(*iter);
+        }
+      }
+#endif
+      return JVMCI::cache_full;
+    }
+    if (stubs_to_free == nullptr) {
+      return JVMCI::ok;
+    }
+    stubs_to_free->append(stub);
+  } while (true);
+}
+
 JVMCI::CodeInstallResult CodeInstaller::install(JVMCICompiler* compiler,
     jlong compiled_code_buffer,
     bool with_type_info,
@@ -707,17 +755,7 @@ JVMCI::CodeInstallResult CodeInstaller::install(JVMCICompiler* compiler,
   int stack_slots = _total_frame_size / HeapWordSize; // conversion to words
 
   if (!is_nmethod) {
-    if (name == nullptr) {
-      JVMCI_ERROR_OK("stub should have a name");
-    }
-    name = os::strdup(name); // Note: this leaks. See JDK-8289632
-    cb = RuntimeStub::new_runtime_stub(name,
-                                       &buffer,
-                                       _offsets.value(CodeOffsets::Frame_Complete),
-                                       stack_slots,
-                                       _debug_recorder->_oopmaps,
-                                       false);
-    result = JVMCI::ok;
+    return install_runtime_stub(cb, name, &buffer, stack_slots, JVMCI_CHECK_OK);
   } else {
     if (compile_state != nullptr) {
       jvmci_env()->set_compile_state(compile_state);

src/hotspot/share/jvmci/jvmciCodeInstaller.hpp

@@ -399,6 +399,12 @@ class CodeInstaller : public StackObj {
   void read_virtual_objects(HotSpotCompiledCodeStream* stream, JVMCI_TRAPS);
 
   int estimateStubSpace(int static_call_stubs);
+
+  JVMCI::CodeInstallResult install_runtime_stub(CodeBlob*& cb,
+                                                const char* name,
+                                                CodeBuffer* buffer,
+                                                int stack_slots,
+                                                JVMCI_TRAPS);
 };
 
 #endif // SHARE_JVMCI_JVMCICODEINSTALLER_HPP

test/hotspot/jtreg/compiler/jvmci/events/JvmciNotifyInstallEventTest.java

@@ -48,7 +48,9 @@
  *      compiler.jvmci.common.JVMCIHelpers$EmptyVMEventListener
  * @run main/othervm -XX:+UnlockExperimentalVMOptions
  *     -Djvmci.Compiler=EmptyCompiler -Xbootclasspath/a:.
- *     -XX:+UseJVMCICompiler -XX:-BootstrapJVMCI -XX:-UseJVMCINativeLibrary
+ *     -XX:+UseJVMCICompiler -XX:-BootstrapJVMCI
+ *     -XX:-UseJVMCINativeLibrary -XX:JVMCITraceLevel=1
+ *     -Dtest.jvmci.forceRuntimeStubAllocFail=test_stub_that_fails_to_be_allocated
  *     compiler.jvmci.events.JvmciNotifyInstallEventTest
  * @run main/othervm -XX:+UnlockExperimentalVMOptions
  *     -Djvmci.Compiler=EmptyCompiler -Xbootclasspath/a:.
@@ -62,8 +64,10 @@
 import compiler.jvmci.common.CTVMUtilities;
 import compiler.jvmci.common.testcases.SimpleClass;
 import jdk.test.lib.Asserts;
+import jdk.test.lib.Platform;
 import jdk.test.lib.Utils;
 import jdk.vm.ci.services.JVMCIServiceLocator;
+import jdk.vm.ci.code.BailoutException;
 import jdk.vm.ci.code.CompiledCode;
 import jdk.vm.ci.code.InstalledCode;
 import jdk.vm.ci.code.site.DataPatch;
@@ -142,6 +146,29 @@ private void runTest() {
         }, NullPointerException.class);
         Asserts.assertEQ(gotInstallNotification, 2,
                 "Got unexpected event count after 4th install attempt");
+
+        String stubToFail = System.getProperty("test.jvmci.forceRuntimeStubAllocFail");
+        if (Platform.isDebugBuild() && stubToFail != null) {
+            HotSpotCompiledCode stub = new HotSpotCompiledCode(stubToFail,
+                    /* targetCode */ new byte[0],
+                    /* targetCodeSize */ 0,
+                    /* sites */ new Site[0],
+                    /* assumptions */ new Assumption[0],
+                    /* methods */ new ResolvedJavaMethod[0],
+                    /* comments */ new Comment[0],
+                    /* dataSection */ new byte[0],
+                    dataSectionAlignment,
+                    /* dataSectionPatches */ new DataPatch[0],
+                    /* isImmutablePIC */ false,
+                    /* totalFrameSize */ 0,
+                    /* deoptRescueSlot */ null);
+            try {
+                codeCache.installCode(null, stub, null, null, true);
+                throw new AssertionError("Didn't get expected " + BailoutException.class.getName());
+            } catch (BailoutException e) {
+                Asserts.assertEQ(e.getMessage(), "Error installing " + stubToFail + ": code cache is full");
+            }
+        }
     }
 
     @Override