8357135: java.lang.OutOfMemoryError: Error creating or attaching to libjvmci after JDK-8356447

src/hotspot/share/jvmci/jvmci.cpp

@@ -151,7 +151,7 @@ void* JVMCI::get_shared_library(char*& path, bool load) {
   return _shared_library_handle;
 }
 
-void JVMCI::initialize_compiler_in_create_vm(TRAPS) {
+void JVMCI::initialize_compiler(TRAPS) {
   if (JVMCILibDumpJNIConfig) {
     JNIJVMCI::initialize_ids(nullptr);
     ShouldNotReachHere();
@@ -162,24 +162,7 @@ void JVMCI::initialize_compiler_in_create_vm(TRAPS) {
   } else {
       runtime = JVMCI::java_runtime();
   }
-
-  // Enter a JVMCI env, which will load libjvmci if it's in use
-  JVMCIENV_FROM_THREAD(THREAD);
-  int init_error = JVMCIENV->init_error();
-  if (init_error != JNI_OK) {
-    if (PrintCompilation) {
-      const char* msg = JVMCIENV->init_error_msg();
-      tty->print_cr("COMPILER INIT ERROR: Error creating or attaching to libjvmci (err: %d, description: %s)",
-        init_error, msg == nullptr ? "unknown" : msg);
-    }
-    return;
-  }
-
-  // Failures in the calls below will propagate to the caller
-  // and cause VM to exit.
-  JVMCIObject jvmciRuntime = runtime->get_HotSpotJVMCIRuntime(JVMCI_CHECK);
-  runtime->initialize(JVMCI_CHECK);
-  JVMCIENV->call_HotSpotJVMCIRuntime_getCompiler(jvmciRuntime, JVMCI_CHECK);
+  runtime->call_getCompiler(CHECK);
 }
 
 void JVMCI::initialize_globals() {

src/hotspot/share/jvmci/jvmci.hpp

@@ -191,8 +191,9 @@ class JVMCI : public AllStatic {
 
   static void initialize_globals();
 
-  // Called to initialize the JVMCI compiler during VM startup.
-  static void initialize_compiler_in_create_vm(TRAPS);
+  // Called to force initialization of the JVMCI compiler
+  // early in VM startup.
+  static void initialize_compiler(TRAPS);
 
   // Ensures the boxing cache classes (e.g., java.lang.Integer.IntegerCache) are initialized.
   static void ensure_box_caches_initialized(TRAPS);

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -247,6 +247,19 @@ void JVMCIEnv::check_init(JVMCI_TRAPS) {
   JVMCI_THROW_MSG(InternalError, st.freeze());
 }
 
+void JVMCIEnv::check_init(TRAPS) {
+  if (_init_error == JNI_OK) {
+    return;
+  }
+  if (_init_error == JNI_ENOMEM) {
+    THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), "JNI_ENOMEM creating or attaching to libjvmci");
+  }
+  stringStream st;
+  st.print("Error creating or attaching to libjvmci (err: %d, description: %s)",
+           _init_error, _init_error_msg == nullptr ? "unknown" : _init_error_msg);
+  THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), st.freeze());
+}
+
 // Prints a pending exception (if any) and its stack trace to st.
 // Also partially logs the stack trace to the JVMCI event log.
 void JVMCIEnv::describe_pending_exception(outputStream* st) {

src/hotspot/share/jvmci/jvmciEnv.hpp

@@ -243,6 +243,10 @@ class JVMCIEnv : public ResourceObj {
   // (which must not be this) if it is not JNI_OK.
   void check_init(JVMCI_TRAPS);
 
+  // Checks the value of init_error() and throws an exception in `TRAPS`
+  // if it is not JNI_OK.
+  void check_init(TRAPS);
+
   JVMCIRuntime* runtime() {
     guarantee(_init_error == 0, "invalid JVMCIEnv: %d", _init_error);
     return _runtime;

src/hotspot/share/jvmci/jvmciRuntime.cpp

@@ -743,6 +743,14 @@ JVM_ENTRY_NO_ENV(jlong, JVM_ReadSystemPropertiesInfo(JNIEnv *env, jclass c, jint
 JVM_END
 
 
+void JVMCIRuntime::call_getCompiler(TRAPS) {
+  JVMCIENV_FROM_THREAD(THREAD);
+  JVMCIENV->check_init(CHECK);
+  JVMCIObject jvmciRuntime = JVMCIRuntime::get_HotSpotJVMCIRuntime(JVMCI_CHECK);
+  initialize(JVMCI_CHECK);
+  JVMCIENV->call_HotSpotJVMCIRuntime_getCompiler(jvmciRuntime, JVMCI_CHECK);
+}
+
 void JVMCINMethodData::initialize(int nmethod_mirror_index,
                                   int nmethod_entry_patch_offset,
                                   const char* nmethod_mirror_name,
@@ -1592,9 +1600,8 @@ bool JVMCIRuntime::destroy_shared_library_javavm() {
 void JVMCIRuntime::bootstrap_finished(TRAPS) {
   if (_HotSpotJVMCIRuntime_instance.is_non_null()) {
     JVMCIENV_FROM_THREAD(THREAD);
-    if (JVMCIENV->init_error() == JNI_OK) {
-      JVMCIENV->call_HotSpotJVMCIRuntime_bootstrapFinished(_HotSpotJVMCIRuntime_instance, JVMCIENV);
-    }
+    JVMCIENV->check_init(CHECK);
+    JVMCIENV->call_HotSpotJVMCIRuntime_bootstrapFinished(_HotSpotJVMCIRuntime_instance, JVMCIENV);
   }
 }
 

src/hotspot/share/jvmci/jvmciRuntime.hpp

@@ -373,6 +373,8 @@ class JVMCIRuntime: public CHeapObj<mtJVMCI> {
   // Explicitly initialize HotSpotJVMCIRuntime itself
   void initialize_HotSpotJVMCIRuntime(JVMCI_TRAPS);
 
+  void call_getCompiler(TRAPS);
+
   // Shuts down this runtime by calling HotSpotJVMCIRuntime.shutdown().
   // If this is the last thread attached to this runtime, then
   // `_HotSpotJVMCIRuntime_instance` is set to null and `_init_state`

src/hotspot/share/runtime/threads.cpp

@@ -835,7 +835,7 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
 
 #if INCLUDE_JVMCI
   if (force_JVMCI_initialization) {
-    JVMCI::initialize_compiler_in_create_vm(CHECK_JNI_ERR);
+    JVMCI::initialize_compiler(CHECK_JNI_ERR);
   }
 #endif
 

test/hotspot/jtreg/gc/arguments/TestUseCompressedOopsFlagsWithUlimit.java

@@ -54,6 +54,7 @@ private static void checkFlag(long ulimit, long maxram, int maxrampercent, boole
     args.add("-XX:MaxRAM=" + maxram);
     args.add("-XX:MaxRAMPercentage=" + maxrampercent);
     args.add("-XX:+PrintFlagsFinal");
+    args.add("-Xint");
     args.add("-version");
 
     // Convert bytes to kbytes for ulimit -v

test/hotspot/jtreg/vmTestbase/nsk/jvmti/Allocate/alloc001/alloc001.java

@@ -85,6 +85,7 @@ public static void main(String[] args) throws Throwable {
                 "-Djava.library.path=" + Utils.TEST_NATIVE_PATH,
                 "-agentpath:" + Utils.TEST_NATIVE_PATH + File.separator + System.mapLibraryName("alloc001"),
                 "-XX:CompressedClassSpaceSize=64m",
+                "-Xint",
                 Test.class.getName()
         ));
         cmd = escapeCmd(cmd);