8357135: java.lang.OutOfMemoryError: Error creating or attaching to libjvmci after JDK-8356447

src/hotspot/share/jvmci/jvmci.cpp

@@ -151,7 +151,7 @@ void* JVMCI::get_shared_library(char*& path, bool load) {
   return _shared_library_handle;
 }
 
-void JVMCI::initialize_compiler(TRAPS) {
+void JVMCI::initialize_compiler_in_create_vm(TRAPS) {
   if (JVMCILibDumpJNIConfig) {
     JNIJVMCI::initialize_ids(nullptr);
     ShouldNotReachHere();
@@ -162,7 +162,24 @@ void JVMCI::initialize_compiler(TRAPS) {
   } else {
       runtime = JVMCI::java_runtime();
   }
-  runtime->call_getCompiler(CHECK);
+
+  // Enter a JVMCI env, which will load libjvmci if it's in use
+  JVMCIENV_FROM_THREAD(THREAD);
+  int init_error = JVMCIENV->init_error();
+  if (init_error != JNI_OK) {
+    if (PrintCompilation) {
+      const char* msg = JVMCIENV->init_error_msg();
+      tty->print_cr("COMPILER INIT ERROR: Error creating or attaching to libjvmci (err: %d, description: %s)",
+        init_error, msg == nullptr ? "unknown" : msg);
+    }
+    return;
+  }
+
+  // Failures in the calls below will propagate to the caller
+  // and cause VM to exit.
+  JVMCIObject jvmciRuntime = runtime->get_HotSpotJVMCIRuntime(JVMCI_CHECK);
+  runtime->initialize(JVMCI_CHECK);
+  JVMCIENV->call_HotSpotJVMCIRuntime_getCompiler(jvmciRuntime, JVMCI_CHECK);
 }
 
 void JVMCI::initialize_globals() {

src/hotspot/share/jvmci/jvmci.hpp

@@ -191,9 +191,8 @@ class JVMCI : public AllStatic {
 
   static void initialize_globals();
 
-  // Called to force initialization of the JVMCI compiler
-  // early in VM startup.
-  static void initialize_compiler(TRAPS);
+  // Called to initialize the JVMCI compiler during VM startup.
+  static void initialize_compiler_in_create_vm(TRAPS);
 
   // Ensures the boxing cache classes (e.g., java.lang.Integer.IntegerCache) are initialized.
   static void ensure_box_caches_initialized(TRAPS);

src/hotspot/share/jvmci/jvmciRuntime.cpp

@@ -743,22 +743,6 @@ JVM_ENTRY_NO_ENV(jlong, JVM_ReadSystemPropertiesInfo(JNIEnv *env, jclass c, jint
 JVM_END
 
 
-void JVMCIRuntime::call_getCompiler(TRAPS) {
-  JVMCIENV_FROM_THREAD(THREAD);
-  int init_error = JVMCIENV->init_error();
-  if (init_error != JNI_OK) {
-    if (PrintCompilation) {
-      const char* msg = JVMCIENV->init_error_msg();
-      tty->print_cr("COMPILER INIT ERROR: Error creating or attaching to libjvmci (err: %d, description: %s)",
-        init_error, msg == nullptr ? "unknown" : msg);
-    }
-    return;
-  }
-  JVMCIObject jvmciRuntime = JVMCIRuntime::get_HotSpotJVMCIRuntime(JVMCI_CHECK);
-  initialize(JVMCI_CHECK);
-  JVMCIENV->call_HotSpotJVMCIRuntime_getCompiler(jvmciRuntime, JVMCI_CHECK);
-}
-
 void JVMCINMethodData::initialize(int nmethod_mirror_index,
                                   int nmethod_entry_patch_offset,
                                   const char* nmethod_mirror_name,

src/hotspot/share/jvmci/jvmciRuntime.hpp

@@ -373,8 +373,6 @@ class JVMCIRuntime: public CHeapObj<mtJVMCI> {
   // Explicitly initialize HotSpotJVMCIRuntime itself
   void initialize_HotSpotJVMCIRuntime(JVMCI_TRAPS);
 
-  void call_getCompiler(TRAPS);
-
   // Shuts down this runtime by calling HotSpotJVMCIRuntime.shutdown().
   // If this is the last thread attached to this runtime, then
   // `_HotSpotJVMCIRuntime_instance` is set to null and `_init_state`

src/hotspot/share/runtime/threads.cpp

@@ -835,7 +835,7 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
 
 #if INCLUDE_JVMCI
   if (force_JVMCI_initialization) {
-    JVMCI::initialize_compiler(CHECK_JNI_ERR);
+    JVMCI::initialize_compiler_in_create_vm(CHECK_JNI_ERR);
   }
 #endif