8315369: [JVMCI] failure to attach to a libgraal isolate during shutdown should not be fatal

Reviewed-by: never

Author: Doug Simon

src/hotspot/share/compiler/compileBroker.cpp

@@ -2203,7 +2203,14 @@ void CompileBroker::invoke_compiler_on_method(CompileTask* task) {
       compilable = ciEnv::MethodCompilable_never;
     } else {
       JVMCIEnv env(thread, &compile_state, __FILE__, __LINE__);
-      failure_reason = compile_state.failure_reason();
+      if (env.init_error() != JNI_OK) {
+        failure_reason = os::strdup(err_msg("Error attaching to libjvmci (err: %d)", env.init_error()), mtJVMCI);
+        bool reason_on_C_heap = true;
+        // In case of JNI_ENOMEM, there's a good chance a subsequent attempt to create libjvmci or attach to it
+        // might succeed. Other errors most likely indicate a non-recoverable error in the JVMCI runtime.
+        bool retryable = env.init_error() == JNI_ENOMEM;
+        compile_state.set_failure(retryable, failure_reason, reason_on_C_heap);
+      }
       if (failure_reason == nullptr) {
         if (WhiteBoxAPI && WhiteBox::compilation_locked) {
           // Must switch to native to block

src/hotspot/share/jvmci/jniAccessMark.inline.hpp

@@ -44,6 +44,7 @@ class JNIAccessMark : public StackObj {
   inline JNIAccessMark(JVMCIEnv* jvmci_env, JavaThread* thread=JavaThread::current()) :
     _ttnfv(thread), _hm(thread) {
     _env = jvmci_env->_env;
+    guarantee(jvmci_env->init_error() == JNI_OK, "invalid JVMCIEnv (err: %d)", jvmci_env->init_error());
   }
   JNIEnv* env() const { return _env; }
   JNIEnv* operator () () const { return _env; }

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -171,7 +171,7 @@ Handle JavaArgumentUnboxer::next_arg(BasicType expectedType) {
 #define C2V_BLOCK(result_type, name, signature)      \
   JVMCI_VM_ENTRY_MARK;                               \
   ResourceMark rm;                                   \
-  JNI_JVMCIENV(JVMCI::compilation_tick(thread), env);
+  JVMCIENV_FROM_JNI(JVMCI::compilation_tick(thread), env);
 
 static JavaThread* get_current_thread(bool allow_null=true) {
   Thread* thread = Thread::current_or_null_safe();
@@ -2436,16 +2436,13 @@ C2V_VMENTRY_NULL(jlongArray, registerNativeMethods, (JNIEnv* env, jobject, jclas
   JVMCIRuntime* runtime;
   {
     // Ensure the JVMCI shared library runtime is initialized.
-    bool jni_enomem_is_fatal = false;
-    JVMCIEnv __peer_jvmci_env__(thread, false, jni_enomem_is_fatal, __FILE__, __LINE__);
-    JVMCIEnv* peerEnv = &__peer_jvmci_env__;
-    if (peerEnv->has_jni_enomem()) {
-      JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
-    }
+    PEER_JVMCIENV_FROM_THREAD(THREAD, false);
+    PEER_JVMCIENV->check_init(JVMCI_CHECK_NULL);
+
     HandleMark hm(THREAD);
     runtime = JVMCI::compiler_runtime(thread);
-    if (peerEnv->has_pending_exception()) {
-      peerEnv->describe_pending_exception(tty);
+    if (PEER_JVMCIENV->has_pending_exception()) {
+      PEER_JVMCIENV->describe_pending_exception(tty);
     }
     sl_handle = JVMCI::get_shared_library(sl_path, false);
     if (sl_handle == nullptr) {
@@ -2604,17 +2601,13 @@ C2V_VMENTRY_PREFIX(jboolean, attachCurrentThread, (JNIEnv* env, jobject c2vm, jb
 
     {
       // Ensure the JVMCI shared library runtime is initialized.
-      bool jni_enomem_is_fatal = false;
-      JVMCIEnv __peer_jvmci_env__(thread, false, jni_enomem_is_fatal, __FILE__, __LINE__);
-      JVMCIEnv* peerJVMCIEnv = &__peer_jvmci_env__;
-      if (peerJVMCIEnv->has_jni_enomem()) {
-        JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
-      }
+      PEER_JVMCIENV_FROM_THREAD(THREAD, false);
+      PEER_JVMCIENV->check_init(JVMCI_CHECK_0);
 
       HandleMark hm(thread);
-      JVMCIObject receiver = runtime->get_HotSpotJVMCIRuntime(peerJVMCIEnv);
-      if (peerJVMCIEnv->has_pending_exception()) {
-        peerJVMCIEnv->describe_pending_exception(tty);
+      JVMCIObject receiver = runtime->get_HotSpotJVMCIRuntime(PEER_JVMCIENV);
+      if (PEER_JVMCIENV->has_pending_exception()) {
+        PEER_JVMCIENV->describe_pending_exception(tty);
       }
       char* sl_path;
       if (JVMCI::get_shared_library(sl_path, false) == nullptr) {
@@ -2704,33 +2697,29 @@ C2V_VMENTRY_0(jlong, translate, (JNIEnv* env, jobject, jobject obj_handle, jbool
   if (obj_handle == nullptr) {
     return 0L;
   }
-  bool jni_enomem_is_fatal = false;
-  JVMCIEnv __peer_jvmci_env__(thread, !JVMCIENV->is_hotspot(), jni_enomem_is_fatal, __FILE__, __LINE__);
-  JVMCIEnv* peerEnv = &__peer_jvmci_env__;
-  JVMCIEnv* thisEnv = JVMCIENV;
-  if (peerEnv->has_jni_enomem()) {
-      JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
-  }
+  PEER_JVMCIENV_FROM_THREAD(THREAD, !JVMCIENV->is_hotspot());
+  PEER_JVMCIENV->check_init(JVMCI_CHECK_0);
 
+  JVMCIEnv* thisEnv = JVMCIENV;
   JVMCIObject obj = thisEnv->wrap(obj_handle);
   JVMCIObject result;
   if (thisEnv->isa_HotSpotResolvedJavaMethodImpl(obj)) {
     methodHandle method(THREAD, thisEnv->asMethod(obj));
-    result = peerEnv->get_jvmci_method(method, JVMCI_CHECK_0);
+    result = PEER_JVMCIENV->get_jvmci_method(method, JVMCI_CHECK_0);
   } else if (thisEnv->isa_HotSpotResolvedObjectTypeImpl(obj)) {
     Klass* klass = thisEnv->asKlass(obj);
     JVMCIKlassHandle klass_handle(THREAD);
     klass_handle = klass;
-    result = peerEnv->get_jvmci_type(klass_handle, JVMCI_CHECK_0);
+    result = PEER_JVMCIENV->get_jvmci_type(klass_handle, JVMCI_CHECK_0);
   } else if (thisEnv->isa_HotSpotResolvedPrimitiveType(obj)) {
     BasicType type = JVMCIENV->kindToBasicType(JVMCIENV->get_HotSpotResolvedPrimitiveType_kind(obj), JVMCI_CHECK_0);
-    result = peerEnv->get_jvmci_primitive_type(type);
+    result = PEER_JVMCIENV->get_jvmci_primitive_type(type);
   } else if (thisEnv->isa_IndirectHotSpotObjectConstantImpl(obj) ||
              thisEnv->isa_DirectHotSpotObjectConstantImpl(obj)) {
     Handle constant = thisEnv->asConstant(obj, JVMCI_CHECK_0);
-    result = peerEnv->get_object_constant(constant());
+    result = PEER_JVMCIENV->get_object_constant(constant());
   } else if (thisEnv->isa_HotSpotNmethod(obj)) {
-    if (peerEnv->is_hotspot()) {
+    if (PEER_JVMCIENV->is_hotspot()) {
       nmethod* nm = JVMCIENV->get_nmethod(obj);
       if (nm != nullptr) {
         JVMCINMethodData* data = nm->jvmci_nmethod_data();
@@ -2753,17 +2742,17 @@ C2V_VMENTRY_0(jlong, translate, (JNIEnv* env, jobject, jobject obj_handle, jbool
       JVMCIObject name_string = thisEnv->get_InstalledCode_name(obj);
       const char* cstring = name_string.is_null() ? nullptr : thisEnv->as_utf8_string(name_string);
       // Create a new HotSpotNmethod instance in the peer runtime
-      result = peerEnv->new_HotSpotNmethod(mh, cstring, isDefault, compileIdSnapshot, JVMCI_CHECK_0);
+      result = PEER_JVMCIENV->new_HotSpotNmethod(mh, cstring, isDefault, compileIdSnapshot, JVMCI_CHECK_0);
       nmethod* nm = JVMCIENV->get_nmethod(obj);
       if (result.is_null()) {
         // exception occurred (e.g. OOME) creating a new HotSpotNmethod
       } else if (nm == nullptr) {
         // nmethod must have been unloaded
       } else {
         // Link the new HotSpotNmethod to the nmethod
-        peerEnv->initialize_installed_code(result, nm, JVMCI_CHECK_0);
+        PEER_JVMCIENV->initialize_installed_code(result, nm, JVMCI_CHECK_0);
         // Only HotSpotNmethod instances in the HotSpot heap are tracked directly by the runtime.
-        if (peerEnv->is_hotspot()) {
+        if (PEER_JVMCIENV->is_hotspot()) {
           JVMCINMethodData* data = nm->jvmci_nmethod_data();
           if (data == nullptr) {
             JVMCI_THROW_MSG_0(IllegalArgumentException, "Cannot set HotSpotNmethod mirror for default nmethod");
@@ -2781,13 +2770,13 @@ C2V_VMENTRY_0(jlong, translate, (JNIEnv* env, jobject, jobject obj_handle, jbool
                 err_msg("Cannot translate object of type: %s", thisEnv->klass_name(obj)));
   }
   if (callPostTranslation) {
-    peerEnv->call_HotSpotJVMCIRuntime_postTranslation(result, JVMCI_CHECK_0);
+    PEER_JVMCIENV->call_HotSpotJVMCIRuntime_postTranslation(result, JVMCI_CHECK_0);
   }
   // Propagate any exception that occurred while creating the translated object
-  if (peerEnv->transfer_pending_exception(thread, thisEnv)) {
+  if (PEER_JVMCIENV->transfer_pending_exception(thread, thisEnv)) {
     return 0L;
   }
-  return (jlong) peerEnv->make_global(result).as_jobject();
+  return (jlong) PEER_JVMCIENV->make_global(result).as_jobject();
 }
 
 C2V_VMENTRY_NULL(jobject, unhand, (JNIEnv* env, jobject, jlong obj_handle))

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -115,7 +115,7 @@ bool JVMCICompileState::jvmti_state_changed() const {
   return false;
 }
 
-void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool jni_enomem_is_fatal) {
+void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env) {
   assert(thread != nullptr, "npe");
   _env = nullptr;
   _pop_frame_on_close = false;
@@ -147,18 +147,14 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
   _is_hotspot = false;
 
   _runtime = JVMCI::compiler_runtime(thread);
-  int create_JavaVM_err = JNI_OK;
-  _env = _runtime->init_shared_library_javavm(&create_JavaVM_err);
+  _env = _runtime->init_shared_library_javavm(&_init_error);
   if (_env != nullptr) {
     // Creating the JVMCI shared library VM also attaches the current thread
     _detach_on_close = true;
-  } else if (create_JavaVM_err != JNI_OK) {
-    if (!jni_enomem_is_fatal && create_JavaVM_err == JNI_ENOMEM) {
-      _jni_enomem = true;
-      return;
-    } else {
-      fatal("JNI_CreateJavaVM failed with return value %d", create_JavaVM_err);
-    }
+  } else if (_init_error != JNI_OK) {
+    // Caller creating this JVMCIEnv must handle the error.
+    JVMCI_event_1("[%s:%d] Error creating libjvmci (err: %d)", _file, _line, _init_error);
+    return;
   } else {
     _runtime->GetEnv(thread, (void**)&parent_env, JNI_VERSION_1_2);
     if (parent_env != nullptr) {
@@ -174,15 +170,14 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
       attach_args.version = JNI_VERSION_1_2;
       attach_args.name = const_cast<char*>(thread->name());
       attach_args.group = nullptr;
-      jint attach_result = _runtime->AttachCurrentThread(thread, (void**) &_env, &attach_args);
-      if (attach_result == JNI_OK) {
+      _init_error = _runtime->AttachCurrentThread(thread, (void**) &_env, &attach_args);
+      if (_init_error == JNI_OK) {
         _detach_on_close = true;
-      } else if (!jni_enomem_is_fatal && attach_result == JNI_ENOMEM) {
+      } else {
+        // Caller creating this JVMCIEnv must handle the error.
         _env = nullptr;
-        _jni_enomem = true;
+        JVMCI_event_1("[%s:%d] Error attaching to libjvmci (err: %d)", _file, _line, _init_error);
         return;
-      } else {
-        fatal("Error attaching current thread (%s) to JVMCI shared library JNI interface", attach_args.name);
       }
     }
   }
@@ -193,50 +188,66 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
   JNIAccessMark jni(this, thread);
   jint result = _env->PushLocalFrame(32);
   if (result != JNI_OK) {
-    char message[256];
-    jio_snprintf(message, 256, "Uncaught exception pushing local frame for JVMCIEnv scope entered at %s:%d", _file, _line);
-    JVMCIRuntime::fatal_exception(this, message);
+    JVMCI_event_1("[%s:%d] Error pushing local JNI frame (err: %d)", _file, _line, _init_error);
+    return;
   }
   _pop_frame_on_close = true;
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JVMCICompileState* compile_state, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _jni_enomem(false), _compile_state(compile_state) {
-  // In case of JNI_ENOMEM, there's a good chance a subsequent attempt to create libjvmci or attach to it
-  // might succeed. Other errors most likely indicate a non-recoverable error in the JVMCI runtime.
-  bool jni_enomem_is_fatal = false;
-  init_env_mode_runtime(thread, nullptr, jni_enomem_is_fatal);
-  if (_jni_enomem) {
-    compile_state->set_failure(true, "Out of memory while attaching JVMCI compiler to current thread");
-  }
+    _throw_to_caller(false), _file(file), _line(line), _init_error(JNI_OK), _compile_state(compile_state) {
+  init_env_mode_runtime(thread, nullptr);
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _jni_enomem(false), _compile_state(nullptr) {
+    _throw_to_caller(false), _file(file), _line(line), _init_error(JNI_OK), _compile_state(nullptr) {
   init_env_mode_runtime(thread, nullptr);
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JNIEnv* parent_env, const char* file, int line):
-    _throw_to_caller(true), _file(file), _line(line), _jni_enomem(false), _compile_state(nullptr) {
+    _throw_to_caller(true), _file(file), _line(line), _init_error(JNI_OK), _compile_state(nullptr) {
+  assert(parent_env != nullptr, "npe");
   init_env_mode_runtime(thread, parent_env);
   assert(_env == nullptr || parent_env == _env, "mismatched JNIEnvironment");
+  assert(_init_error == JNI_OK, "err: %d", _init_error);
 }
 
-void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line) {
+void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, const char* file, int line) {
   _compile_state = nullptr;
   _throw_to_caller = false;
   _file = file;
   _line = line;
-  _jni_enomem = false;
+  _init_error = JNI_OK;
   if (is_hotspot) {
     _env = nullptr;
     _pop_frame_on_close = false;
     _detach_on_close = false;
     _is_hotspot = true;
     _runtime = JVMCI::java_runtime();
   } else {
-    init_env_mode_runtime(thread, nullptr, jni_enomem_is_fatal);
+    init_env_mode_runtime(thread, nullptr);
+  }
+}
+
+void JVMCIEnv::check_init(JVMCI_TRAPS) {
+  guarantee(JVMCIENV != this, "must be");
+  if (_init_error == JNI_OK) {
+    return;
+  }
+  if (_init_error == JNI_ENOMEM) {
+    JVMCI_THROW_MSG(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
+  }
+  JVMCI_THROW_MSG(InternalError, err_msg("Error creating or attaching to libjvmci (err: %d)", _init_error));
+}
+
+void JVMCIEnv::check_init(TRAPS) {
+  if (_init_error == JNI_OK) {
+    return;
+  }
+  if (_init_error == JNI_ENOMEM) {
+    THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), "JNI_ENOMEM creating or attaching to libjvmci");
   }
+  THROW_MSG(vmSymbols::java_lang_OutOfMemoryError(), err_msg("Error creating or attaching to libjvmci (err: %d)", _init_error));
 }
 
 // Prints a pending exception (if any) and its stack trace to st.
@@ -561,7 +572,7 @@ jboolean JVMCIEnv::transfer_pending_exception(JavaThread* THREAD, JVMCIEnv* peer
 }
 
 JVMCIEnv::~JVMCIEnv() {
-  if (_jni_enomem) {
+  if (_init_error != JNI_OK) {
     return;
   }
   if (_throw_to_caller) {

src/hotspot/share/jvmci/jvmciEnv.hpp

@@ -157,9 +157,9 @@ class JVMCIEnv : public ResourceObj {
   friend class JNIAccessMark;
 
   // Initializes the _env, _mode and _runtime fields.
-  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool jni_enomem_is_fatal = true);
+  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env);
 
-  void init(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line);
+  void init(JavaThread* thread, bool is_hotspot, const char* file, int line);
 
   JNIEnv*                 _env;  // JNI env for calling into shared library
   bool     _pop_frame_on_close;  // Must pop frame on close?
@@ -169,9 +169,9 @@ class JVMCIEnv : public ResourceObj {
   bool        _throw_to_caller;  // Propagate an exception raised in this env to the caller?
   const char*            _file;  // The file and ...
   int                    _line;  // ... line where this JNIEnv was created
-  bool             _jni_enomem;  // JNI_ENOMEM returned when creating or attaching to a libjvmci isolate.
-                                 // If true, the JVMCIEnv is invalid and should not be used apart from
-                                 // calling has_jni_enomem().
+  int              _init_error;  // JNI code returned when creating or attaching to a libjvmci isolate.
+                                 // If not JNI_OK, the JVMCIEnv is invalid and should not be used apart from
+                                 // calling init_error().
 
   // Translates an exception on the HotSpot heap (i.e., hotspot_env) to an exception on
   // the shared library heap (i.e., jni_env). The translation includes the stack and cause(s) of `throwable`.
@@ -185,11 +185,12 @@ class JVMCIEnv : public ResourceObj {
 
 public:
   // Opens a JVMCIEnv scope for a Java to VM call (e.g., via CompilerToVM).
+  // The `parent_env` argument must not be null.
   // An exception occurring within the scope is left pending when the
   // scope closes so that it will be propagated back to Java.
   // The JVMCIEnv destructor translates the exception object for the
   // Java runtime if necessary.
-  JVMCIEnv(JavaThread* thread, JNIEnv* env, const char* file, int line);
+  JVMCIEnv(JavaThread* thread, JNIEnv* parent_env, const char* file, int line);
 
   // Opens a JVMCIEnv scope for a compilation scheduled by the CompileBroker.
   // An exception occurring within the scope must not be propagated back to
@@ -200,34 +201,32 @@ class JVMCIEnv : public ResourceObj {
   // within the scope must not be propagated back to the caller.
   JVMCIEnv(JavaThread* env, const char* file, int line);
 
-  // Opens a JNIEnv scope for accessing `for_object`. An exception occurring
-  // within the scope must not be propagated back to the caller.
-  JVMCIEnv(JavaThread* thread, JVMCIObject for_object, const char* file, int line) {
-    // A JNI call to access an object in the shared library heap
-    // can block or take a long time so do not allow such access
-    // on the VM thread.
-    assert(for_object.is_hotspot() || !Thread::current()->is_VM_thread(),
-        "cannot open JVMCIEnv scope when in the VM thread for accessing a shared library heap object");
-    bool jni_enomem_is_fatal = true;
-    init(thread, for_object.is_hotspot(), jni_enomem_is_fatal, file, line);
-  }
-
   // Opens a JNIEnv scope for the HotSpot runtime if `is_hotspot` is true
   // otherwise for the shared library runtime. An exception occurring
   // within the scope must not be propagated back to the caller.
-  JVMCIEnv(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line) {
-    init(thread, is_hotspot, jni_enomem_is_fatal, file, line);
+  JVMCIEnv(JavaThread* thread, bool is_hotspot, const char* file, int line) {
+    init(thread, is_hotspot, file, line);
   }
 
   ~JVMCIEnv();
 
-  // Determines if a JNI_ENOMEM occurred while trying to create a libjvmci
-  // isolate or attach to it within the scope of a JVMCIEnv constructor.
-  bool has_jni_enomem() {
-    return _jni_enomem;
+  // Gets the JNI result code returned when creating or attaching to a libjvmci isolate.
+  // If not JNI_OK, the JVMCIEnv is invalid and the caller must abort the operation
+  // this JVMCIEnv context was created for.
+  int init_error() {
+    return _init_error;
   }
 
+  // Checks the value of init_error() and throws an exception in `JVMCI_TRAPS`
+  // (which must not be this) if it is not JNI_OK.
+  void check_init(JVMCI_TRAPS);
+
+  // Checks the value of init_error() and throws an exception in `TRAPS`
+  // if it is not JNI_OK.
+  void check_init(TRAPS);
+
   JVMCIRuntime* runtime() {
+    guarantee(_init_error == 0, "invalid JVMCIEnv: %d", _init_error);
     return _runtime;
   }