8306992: [JVMCI] mitigate more against JVMCI related OOME causing VM to exit

Reviewed-by: never

Author: Doug Simon

src/hotspot/share/jvmci/jvmci.cpp

@@ -233,12 +233,12 @@ void JVMCI::vtrace(int level, const char* format, va_list ap) {
       ResourceMark rm(thread);
       JavaThreadState state = JavaThread::cast(thread)->thread_state();
       if (state == _thread_in_vm || state == _thread_in_Java || state == _thread_new) {
-        tty->print("JVMCITrace-%d[%s]:%*c", level, thread->name(), level, ' ');
+        tty->print("JVMCITrace-%d[" PTR_FORMAT " \"%s\"]:%*c", level, p2i(thread), thread->name(), level, ' ');
       } else {
         // According to check_access_thread_state, it's unsafe to
         // resolve the j.l.Thread object unless the thread is in
         // one of the states above.
-        tty->print("JVMCITrace-%d[%s@" PTR_FORMAT "]:%*c", level, thread->type_name(), p2i(thread), level, ' ');
+        tty->print("JVMCITrace-%d[" PTR_FORMAT " <%s>]:%*c", level, p2i(thread), thread->type_name(), level, ' ');
       }
     } else {
       tty->print("JVMCITrace-%d[?]:%*c", level, level, ' ');

src/hotspot/share/jvmci/jvmciCompiler.cpp

@@ -39,6 +39,9 @@ JVMCICompiler::JVMCICompiler() : AbstractCompiler(compiler_jvmci) {
   _bootstrapping = false;
   _bootstrap_compilation_request_handled = false;
   _methods_compiled = 0;
+  _ok_upcalls = 0;
+  _err_upcalls = 0;
+  _disabled = false;
   _global_compilation_ticks = 0;
   assert(_instance == nullptr, "only one instance allowed");
   _instance = this;
@@ -118,6 +121,9 @@ void JVMCICompiler::bootstrap(TRAPS) {
 }
 
 bool JVMCICompiler::force_comp_at_level_simple(const methodHandle& method) {
+  if (_disabled) {
+    return true;
+  }
   if (_bootstrapping) {
     // When bootstrapping, the JVMCI compiler can compile its own methods.
     return false;
@@ -211,6 +217,39 @@ void JVMCICompiler::inc_methods_compiled() {
   Atomic::inc(&_global_compilation_ticks);
 }
 
+void JVMCICompiler::on_upcall(const char* error, JVMCICompileState* compile_state) {
+  if (error != nullptr) {
+
+    Atomic::inc(&_err_upcalls);
+    int ok = _ok_upcalls;
+    int err = _err_upcalls;
+    // If there have been at least 10 upcalls with an error
+    // and the number of error upcalls is 10% or more of the
+    // number of non-error upcalls, disable JVMCI compilation.
+    if (err > 10 && err * 10 > ok && !_disabled) {
+      _disabled = true;
+      int total = err + ok;
+      const char* disable_msg = err_msg("JVMCI compiler disabled "
+      "after %d of %d upcalls had errors (Last error: \"%s\"). "
+      "Use -Xlog:jit+compilation for more detail.", err, total, error);
+      log_warning(jit,compilation)("%s", disable_msg);
+      if (compile_state != nullptr) {
+        const char* disable_error = os::strdup(disable_msg);
+        if (disable_error != nullptr) {
+          compile_state->set_failure(true, disable_error, true);
+          JVMCI_event_1("%s", disable_error);
+          return;
+        } else {
+          // Leave failure reason as set by caller when strdup fails
+        }
+      }
+    }
+    JVMCI_event_1("JVMCI upcall had an error: %s", error);
+  } else {
+    Atomic::inc(&_ok_upcalls);
+  }
+}
+
 void JVMCICompiler::inc_global_compilation_ticks() {
   Atomic::inc(&_global_compilation_ticks);
 }

src/hotspot/share/jvmci/jvmciCompiler.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,8 @@
 #include "compiler/compiler_globals.hpp"
 #include "runtime/atomic.hpp"
 
+class JVMCICompileState;
+
 class JVMCICompiler : public AbstractCompiler {
  public:
   // Code installation specific statistics.
@@ -62,10 +64,20 @@ class JVMCICompiler : public AbstractCompiler {
 
   /**
    * Number of methods successfully compiled by a call to
-   * JVMCICompiler::compile_method().
+   * JVMCIRuntime::compile_method().
    */
   volatile int _methods_compiled;
 
+  // Tracks upcalls that should only fail under severe conditions (e.g.
+  // memory pressure) and disables JVMCI compilation if too many fail
+  // with an error. A good example is an OOME thrown
+  // when libgraal calls into the HotSpot heap to get a copy
+  // of the system properties or to translate an exception from
+  // the HotSpot heap to the libgraal heap.
+  volatile int _ok_upcalls;
+  volatile int _err_upcalls;
+  bool _disabled;
+
   // Incremented periodically by JVMCI compiler threads
   // to indicate JVMCI compilation activity.
   volatile int _global_compilation_ticks;
@@ -126,6 +138,11 @@ class JVMCICompiler : public AbstractCompiler {
   int methods_compiled() { return _methods_compiled; }
   void inc_methods_compiled();
 
+  // Called after a JVMCI upcall whose success is a measure of the
+  // JVMCI compiler's health. The value of `error` describes
+  // an error during the upcall, null if no error.
+  void on_upcall(const char* error, JVMCICompileState* compile_state=nullptr);
+
   // Gets a value indicating JVMCI compilation activity on any thread.
   // If successive calls to this method return a different value, then
   // some degree of JVMCI compilation occurred between the calls.

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -2399,8 +2399,12 @@ C2V_VMENTRY_NULL(jlongArray, registerNativeMethods, (JNIEnv* env, jobject, jclas
   JVMCIRuntime* runtime;
   {
     // Ensure the JVMCI shared library runtime is initialized.
-    JVMCIEnv __peer_jvmci_env__(thread, false, __FILE__, __LINE__);
+    bool jni_enomem_is_fatal = false;
+    JVMCIEnv __peer_jvmci_env__(thread, false, jni_enomem_is_fatal, __FILE__, __LINE__);
     JVMCIEnv* peerEnv = &__peer_jvmci_env__;
+    if (peerEnv->has_jni_enomem()) {
+      JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
+    }
     HandleMark hm(THREAD);
     runtime = JVMCI::compiler_runtime(thread);
     if (peerEnv->has_pending_exception()) {
@@ -2563,8 +2567,13 @@ C2V_VMENTRY_PREFIX(jboolean, attachCurrentThread, (JNIEnv* env, jobject c2vm, jb
 
     {
       // Ensure the JVMCI shared library runtime is initialized.
-      JVMCIEnv __peer_jvmci_env__(thread, false, __FILE__, __LINE__);
+      bool jni_enomem_is_fatal = false;
+      JVMCIEnv __peer_jvmci_env__(thread, false, jni_enomem_is_fatal, __FILE__, __LINE__);
       JVMCIEnv* peerJVMCIEnv = &__peer_jvmci_env__;
+      if (peerJVMCIEnv->has_jni_enomem()) {
+        JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
+      }
+
       HandleMark hm(thread);
       JVMCIObject receiver = runtime->get_HotSpotJVMCIRuntime(peerJVMCIEnv);
       if (peerJVMCIEnv->has_pending_exception()) {
@@ -2658,9 +2667,13 @@ C2V_VMENTRY_0(jlong, translate, (JNIEnv* env, jobject, jobject obj_handle, jbool
   if (obj_handle == nullptr) {
     return 0L;
   }
-  JVMCIEnv __peer_jvmci_env__(thread, !JVMCIENV->is_hotspot(), __FILE__, __LINE__);
+  bool jni_enomem_is_fatal = false;
+  JVMCIEnv __peer_jvmci_env__(thread, !JVMCIENV->is_hotspot(), jni_enomem_is_fatal, __FILE__, __LINE__);
   JVMCIEnv* peerEnv = &__peer_jvmci_env__;
   JVMCIEnv* thisEnv = JVMCIENV;
+  if (peerEnv->has_jni_enomem()) {
+      JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
+  }
 
   JVMCIObject obj = thisEnv->wrap(obj_handle);
   JVMCIObject result;

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -67,6 +67,21 @@ JVMCICompileState::JVMCICompileState(CompileTask* task, JVMCICompiler* compiler)
   }
 }
 
+void JVMCICompileState::set_failure(bool retryable, const char* reason, bool reason_on_C_heap) {
+  if (_failure_reason != nullptr && _failure_reason_on_C_heap) {
+      os::free((void*) _failure_reason);
+  }
+  _failure_reason = reason;
+  _failure_reason_on_C_heap = reason_on_C_heap;
+  _retryable = retryable;
+}
+
+void JVMCICompileState::notify_libjvmci_oome() {
+  const char* msg = "Out of memory initializing libjvmci or attaching it to the current thread";
+  set_failure(true, msg);
+  _compiler->on_upcall(msg);
+}
+
 // Update global JVMCI compilation ticks after 512 thread-local JVMCI compilation ticks.
 // This mitigates the overhead of the atomic operation used for the global update.
 #define THREAD_TICKS_PER_GLOBAL_TICKS (2 << 9)
@@ -172,7 +187,7 @@ void JVMCIEnv::copy_saved_properties(jbyte* properties, int properties_len, JVMC
   }
 }
 
-void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool attach_OOME_is_fatal) {
+void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool jni_enomem_is_fatal) {
   assert(thread != nullptr, "npe");
   _env = nullptr;
   _pop_frame_on_close = false;
@@ -204,11 +219,18 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
   _is_hotspot = false;
 
   _runtime = JVMCI::compiler_runtime(thread);
-  _env = _runtime->init_shared_library_javavm();
-
+  int create_JavaVM_err = JNI_OK;
+  _env = _runtime->init_shared_library_javavm(&create_JavaVM_err);
   if (_env != nullptr) {
     // Creating the JVMCI shared library VM also attaches the current thread
     _detach_on_close = true;
+  } else if (create_JavaVM_err != JNI_OK) {
+    if (!jni_enomem_is_fatal && create_JavaVM_err == JNI_ENOMEM) {
+      _jni_enomem = true;
+      return;
+    } else {
+      fatal("JNI_CreateJavaVM failed with return value %d", create_JavaVM_err);
+    }
   } else {
     _runtime->GetEnv(thread, (void**)&parent_env, JNI_VERSION_1_2);
     if (parent_env != nullptr) {
@@ -227,9 +249,9 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
       jint attach_result = _runtime->AttachCurrentThread(thread, (void**) &_env, &attach_args);
       if (attach_result == JNI_OK) {
         _detach_on_close = true;
-      } else if (!attach_OOME_is_fatal && attach_result == JNI_ENOMEM) {
+      } else if (!jni_enomem_is_fatal && attach_result == JNI_ENOMEM) {
         _env = nullptr;
-        _attach_threw_OOME = true;
+        _jni_enomem = true;
         return;
       } else {
         fatal("Error attaching current thread (%s) to JVMCI shared library JNI interface", attach_args.name);
@@ -251,40 +273,41 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JVMCICompileState* compile_state, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(compile_state) {
-  // In case of OOME, there's a good chance a subsequent attempt to attach might succeed.
-  // Other errors most likely indicate a non-recoverable error in the JVMCI runtime.
-  init_env_mode_runtime(thread, nullptr, false);
-  if (_attach_threw_OOME) {
+    _throw_to_caller(false), _file(file), _line(line), _jni_enomem(false), _compile_state(compile_state) {
+  // In case of JNI_ENOMEM, there's a good chance a subsequent attempt to create libjvmci or attach to it
+  // might succeed. Other errors most likely indicate a non-recoverable error in the JVMCI runtime.
+  bool jni_enomem_is_fatal = false;
+  init_env_mode_runtime(thread, nullptr, jni_enomem_is_fatal);
+  if (_jni_enomem) {
     compile_state->set_failure(true, "Out of memory while attaching JVMCI compiler to current thread");
   }
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(nullptr) {
+    _throw_to_caller(false), _file(file), _line(line), _jni_enomem(false), _compile_state(nullptr) {
   init_env_mode_runtime(thread, nullptr);
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JNIEnv* parent_env, const char* file, int line):
-    _throw_to_caller(true), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(nullptr) {
+    _throw_to_caller(true), _file(file), _line(line), _jni_enomem(false), _compile_state(nullptr) {
   init_env_mode_runtime(thread, parent_env);
   assert(_env == nullptr || parent_env == _env, "mismatched JNIEnvironment");
 }
 
-void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, const char* file, int line) {
+void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line) {
   _compile_state = nullptr;
   _throw_to_caller = false;
   _file = file;
   _line = line;
-  _attach_threw_OOME = false;
+  _jni_enomem = false;
   if (is_hotspot) {
     _env = nullptr;
     _pop_frame_on_close = false;
     _detach_on_close = false;
     _is_hotspot = true;
     _runtime = JVMCI::java_runtime();
   } else {
-    init_env_mode_runtime(thread, nullptr);
+    init_env_mode_runtime(thread, nullptr, jni_enomem_is_fatal);
   }
 }
 
@@ -464,7 +487,7 @@ jboolean JVMCIEnv::transfer_pending_exception(JavaThread* THREAD, JVMCIEnv* peer
 }
 
 JVMCIEnv::~JVMCIEnv() {
-  if (_attach_threw_OOME) {
+  if (_jni_enomem) {
     return;
   }
   if (_throw_to_caller) {
@@ -775,6 +798,7 @@ DO_THROW(IllegalArgumentException)
 DO_THROW(InvalidInstalledCodeException)
 DO_THROW(UnsatisfiedLinkError)
 DO_THROW(UnsupportedOperationException)
+DO_THROW(OutOfMemoryError)
 DO_THROW(ClassNotFoundException)
 
 #undef DO_THROW

src/hotspot/share/jvmci/jvmciEnv.hpp

@@ -137,11 +137,11 @@ class JVMCICompileState : public ResourceObj {
   bool failure_reason_on_C_heap() { return _failure_reason_on_C_heap; }
   bool retryable() { return _retryable; }
 
-  void set_failure(bool retryable, const char* reason, bool reason_on_C_heap = false) {
-    _failure_reason = reason;
-    _failure_reason_on_C_heap = reason_on_C_heap;
-    _retryable = retryable;
-  }
+  void set_failure(bool retryable, const char* reason, bool reason_on_C_heap = false);
+
+  // Called when creating or attaching to a libjvmci isolate failed
+  // due to an out of memory condition.
+  void notify_libjvmci_oome();
 
   jint compilation_ticks() const { return _compilation_ticks; }
   void inc_compilation_ticks();
@@ -157,9 +157,9 @@ class JVMCIEnv : public ResourceObj {
   friend class JNIAccessMark;
 
   // Initializes the _env, _mode and _runtime fields.
-  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool attach_OOME_is_fatal = true);
+  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool jni_enomem_is_fatal = true);
 
-  void init(JavaThread* thread, bool is_hotspot, const char* file, int line);
+  void init(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line);
 
   JNIEnv*                 _env;  // JNI env for calling into shared library
   bool     _pop_frame_on_close;  // Must pop frame on close?
@@ -169,7 +169,9 @@ class JVMCIEnv : public ResourceObj {
   bool        _throw_to_caller;  // Propagate an exception raised in this env to the caller?
   const char*            _file;  // The file and ...
   int                    _line;  // ... line where this JNIEnv was created
-  bool    _attach_threw_OOME;    // Failed to attach thread due to OutOfMemoryError, the JVMCIEnv is invalid
+  bool             _jni_enomem;  // JNI_ENOMEM returned when creating or attaching to a libjvmci isolate.
+                                 // If true, the JVMCIEnv is invalid and should not be used apart from
+                                 // calling has_jni_enomem().
 
   // Translates an exception on the HotSpot heap (i.e., hotspot_env) to an exception on
   // the shared library heap (i.e., jni_env). The translation includes the stack and cause(s) of `throwable`.
@@ -212,18 +214,25 @@ class JVMCIEnv : public ResourceObj {
     // on the VM thread.
     assert(for_object.is_hotspot() || !Thread::current()->is_VM_thread(),
         "cannot open JVMCIEnv scope when in the VM thread for accessing a shared library heap object");
-    init(thread, for_object.is_hotspot(), file, line);
+    bool jni_enomem_is_fatal = true;
+    init(thread, for_object.is_hotspot(), jni_enomem_is_fatal, file, line);
   }
 
   // Opens a JNIEnv scope for the HotSpot runtime if `is_hotspot` is true
   // otherwise for the shared library runtime. An exception occurring
   // within the scope must not be propagated back to the caller.
-  JVMCIEnv(JavaThread* thread, bool is_hotspot, const char* file, int line) {
-    init(thread, is_hotspot, file, line);
+  JVMCIEnv(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line) {
+    init(thread, is_hotspot, jni_enomem_is_fatal, file, line);
   }
 
   ~JVMCIEnv();
 
+  // Determines if a JNI_ENOMEM occurred while trying to create a libjvmci
+  // isolate or attach to it within the scope of a JVMCIEnv constructor.
+  bool has_jni_enomem() {
+    return _jni_enomem;
+  }
+
   JVMCIRuntime* runtime() {
     return _runtime;
   }
@@ -249,7 +258,9 @@ class JVMCIEnv : public ResourceObj {
   // Returns true if a pending exception was transferred, false otherwise.
   static jboolean transfer_pending_exception_to_jni(JavaThread* THREAD, JVMCIEnv* hotspot_env, JVMCIEnv* jni_env);
 
-  // Prints an exception and stack trace of a pending exception.
+  // Prints the toString() and stack trace of a pending exception.
+  // If there is no pending exception, this is a nop.
+  // If `clear` is false, the pending exception will remain pending upon return.
   void describe_pending_exception(bool clear);
 
   int get_length(JVMCIArray array);
@@ -356,6 +367,7 @@ class JVMCIEnv : public ResourceObj {
   DO_THROW(InvalidInstalledCodeException)
   DO_THROW(UnsatisfiedLinkError)
   DO_THROW(UnsupportedOperationException)
+  DO_THROW(OutOfMemoryError)
   DO_THROW(ClassNotFoundException)
 
 #undef DO_THROW

src/hotspot/share/jvmci/jvmciJavaClasses.hpp

@@ -244,6 +244,9 @@
   start_class(InternalError, java_lang_InternalError)                                                         \
     jvmci_constructor(InternalError, "(Ljava/lang/String;)V")                                                 \
   end_class                                                                                                   \
+  start_class(OutOfMemoryError, java_lang_OutOfMemoryError)                                                         \
+    jvmci_constructor(OutOfMemoryError, "(Ljava/lang/String;)V")                                                 \
+  end_class                                                                                                   \
   start_class(ClassNotFoundException, java_lang_ClassNotFoundException)                                       \
     jvmci_constructor(ClassNotFoundException, "(Ljava/lang/String;)V")                                        \
   end_class                                                                                                   \