8194309: JNI handle allocation failure not reported correctly

Reviewed-by: kbarrett, coleenp

src/hotspot/share/memory/universe.cpp

@@ -118,6 +118,7 @@ OopHandle Universe::_the_null_sentinel;
 
 // _out_of_memory_errors is an objArray
 enum OutOfMemoryInstance { _oom_java_heap,
+                           _oom_c_heap,
                            _oom_metaspace,
                            _oom_class_metaspace,
                            _oom_array_size,
@@ -585,6 +586,10 @@ oop Universe::out_of_memory_error_java_heap() {
   return gen_out_of_memory_error(out_of_memory_errors()->obj_at(_oom_java_heap));
 }
 
+oop Universe::out_of_memory_error_c_heap() {
+  return gen_out_of_memory_error(out_of_memory_errors()->obj_at(_oom_c_heap));
+}
+
 oop Universe::out_of_memory_error_metaspace() {
   return gen_out_of_memory_error(out_of_memory_errors()->obj_at(_oom_metaspace));
 }
@@ -680,6 +685,9 @@ void Universe::create_preallocated_out_of_memory_errors(TRAPS) {
   Handle msg = java_lang_String::create_from_str("Java heap space", CHECK);
   java_lang_Throwable::set_message(oom_array->obj_at(_oom_java_heap), msg());
 
+  msg = java_lang_String::create_from_str("C heap space", CHECK);
+  java_lang_Throwable::set_message(oom_array->obj_at(_oom_c_heap), msg());
+
   msg = java_lang_String::create_from_str("Metaspace", CHECK);
   java_lang_Throwable::set_message(oom_array->obj_at(_oom_metaspace), msg());
 
@@ -1016,7 +1024,6 @@ bool universe_post_init() {
   Handle msg = java_lang_String::create_from_str("/ by zero", CHECK_false);
   java_lang_Throwable::set_message(Universe::arithmetic_exception_instance(), msg());
 
-
   Universe::initialize_known_methods(CHECK_false);
 
   // This needs to be done before the first scavenge/gc, since

src/hotspot/share/memory/universe.hpp

@@ -313,6 +313,7 @@ class Universe: AllStatic {
   // may or may not have a backtrace. If error has a backtrace then the stack trace is already
   // filled in.
   static oop out_of_memory_error_java_heap();
+  static oop out_of_memory_error_c_heap();
   static oop out_of_memory_error_metaspace();
   static oop out_of_memory_error_class_metaspace();
   static oop out_of_memory_error_array_size();

src/hotspot/share/prims/jni.cpp

@@ -753,7 +753,7 @@ JNI_ENTRY(jobject, jni_NewGlobalRef(JNIEnv *env, jobject ref))
   HOTSPOT_JNI_NEWGLOBALREF_ENTRY(env, ref);
 
   Handle ref_handle(thread, JNIHandles::resolve(ref));
-  jobject ret = JNIHandles::make_global(ref_handle);
+  jobject ret = JNIHandles::make_global(ref_handle, AllocFailStrategy::RETURN_NULL);
 
   HOTSPOT_JNI_NEWGLOBALREF_RETURN(ret);
   return ret;
@@ -797,7 +797,8 @@ JNI_ENTRY(jobject, jni_NewLocalRef(JNIEnv *env, jobject ref))
 
   HOTSPOT_JNI_NEWLOCALREF_ENTRY(env, ref);
 
-  jobject ret = JNIHandles::make_local(THREAD, JNIHandles::resolve(ref));
+  jobject ret = JNIHandles::make_local(THREAD, JNIHandles::resolve(ref),
+                                       AllocFailStrategy::RETURN_NULL);
 
   HOTSPOT_JNI_NEWLOCALREF_RETURN(ret);
   return ret;
@@ -3048,10 +3049,13 @@ JNI_END
 
 JNI_ENTRY(jweak, jni_NewWeakGlobalRef(JNIEnv *env, jobject ref))
   JNIWrapper("jni_NewWeakGlobalRef");
- HOTSPOT_JNI_NEWWEAKGLOBALREF_ENTRY(env, ref);
+  HOTSPOT_JNI_NEWWEAKGLOBALREF_ENTRY(env, ref);
   Handle ref_handle(thread, JNIHandles::resolve(ref));
-  jweak ret = JNIHandles::make_weak_global(ref_handle);
- HOTSPOT_JNI_NEWWEAKGLOBALREF_RETURN(ret);
+  jweak ret = JNIHandles::make_weak_global(ref_handle, AllocFailStrategy::RETURN_NULL);
+  if (ret == NULL) {
+    THROW_OOP_(Universe::out_of_memory_error_c_heap(), NULL);
+  }
+  HOTSPOT_JNI_NEWWEAKGLOBALREF_RETURN(ret);
   return ret;
 JNI_END
 
@@ -3127,6 +3131,12 @@ static bool initializeDirectBufferSupport(JNIEnv* env, JavaThread* thread) {
     directBufferClass     = (jclass) env->NewGlobalRef(directBufferClass);
     directByteBufferClass = (jclass) env->NewGlobalRef(directByteBufferClass);
 
+    // Global refs will be NULL if out-of-memory (no exception is pending)
+    if (bufferClass == NULL || directBufferClass == NULL || directByteBufferClass == NULL) {
+      directBufferSupportInitializeFailed = 1;
+      return false;
+    }
+
     // Get needed field and method IDs
     directByteBufferConstructor = env->GetMethodID(directByteBufferClass, "<init>", "(JI)V");
     if (env->ExceptionCheck()) {

src/hotspot/share/runtime/jniHandles.cpp

@@ -58,15 +58,15 @@ jobject JNIHandles::make_local(oop obj) {
   return make_local(Thread::current(), obj);
 }
 
-
-jobject JNIHandles::make_local(Thread* thread, oop obj) {
+// Used by NewLocalRef which requires NULL on out-of-memory
+jobject JNIHandles::make_local(Thread* thread, oop obj, AllocFailType alloc_failmode) {
   if (obj == NULL) {
     return NULL;                // ignore null handles
   } else {
     assert(oopDesc::is_oop(obj), "not an oop");
     assert(thread->is_Java_thread(), "not a Java thread");
     assert(!current_thread_in_native(), "must not be in native");
-    return thread->active_handles()->allocate_handle(obj);
+    return thread->active_handles()->allocate_handle(obj, alloc_failmode);
   }
 }
 
@@ -102,7 +102,6 @@ jobject JNIHandles::make_global(Handle obj, AllocFailType alloc_failmode) {
   return res;
 }
 
-
 jobject JNIHandles::make_weak_global(Handle obj, AllocFailType alloc_failmode) {
   assert(!Universe::heap()->is_gc_active(), "can't extend the root set during GC");
   assert(!current_thread_in_native(), "must not be in native");
@@ -343,7 +342,7 @@ void JNIHandleBlock::zap() {
 }
 #endif // ASSERT
 
-JNIHandleBlock* JNIHandleBlock::allocate_block(Thread* thread)  {
+JNIHandleBlock* JNIHandleBlock::allocate_block(Thread* thread, AllocFailType alloc_failmode)  {
   assert(thread == NULL || thread == Thread::current(), "sanity check");
   JNIHandleBlock* block;
   // Check the thread-local free list for a block so we don't
@@ -361,7 +360,14 @@ JNIHandleBlock* JNIHandleBlock::allocate_block(Thread* thread)  {
                    Mutex::_no_safepoint_check_flag);
     if (_block_free_list == NULL) {
       // Allocate new block
-      block = new JNIHandleBlock();
+      if (alloc_failmode == AllocFailStrategy::RETURN_NULL) {
+        block = new (std::nothrow) JNIHandleBlock();
+        if (block == NULL) {
+          return NULL;
+        }
+      } else {
+        block = new JNIHandleBlock();
+      }
       _blocks_allocated++;
       block->zap();
       #ifndef PRODUCT
@@ -461,7 +467,7 @@ void JNIHandleBlock::oops_do(OopClosure* f) {
 }
 
 
-jobject JNIHandleBlock::allocate_handle(oop obj) {
+jobject JNIHandleBlock::allocate_handle(oop obj, AllocFailType alloc_failmode) {
   assert(Universe::heap()->is_in(obj), "sanity check");
   if (_top == 0) {
     // This is the first allocation or the initial block got zapped when
@@ -509,7 +515,7 @@ jobject JNIHandleBlock::allocate_handle(oop obj) {
   if (_last->_next != NULL) {
     // update last and retry
     _last = _last->_next;
-    return allocate_handle(obj);
+    return allocate_handle(obj, alloc_failmode);
   }
 
   // No space available, we have to rebuild free list or expand
@@ -520,12 +526,15 @@ jobject JNIHandleBlock::allocate_handle(oop obj) {
     Thread* thread = Thread::current();
     Handle obj_handle(thread, obj);
     // This can block, so we need to preserve obj across call.
-    _last->_next = JNIHandleBlock::allocate_block(thread);
+    _last->_next = JNIHandleBlock::allocate_block(thread, alloc_failmode);
+    if (_last->_next == NULL) {
+      return NULL;
+    }
     _last = _last->_next;
     _allocate_before_rebuild--;
     obj = obj_handle();
   }
-  return allocate_handle(obj);  // retry
+  return allocate_handle(obj, alloc_failmode);  // retry
 }
 
 void JNIHandleBlock::rebuild_free_list() {

src/hotspot/share/runtime/jniHandles.hpp

@@ -84,15 +84,18 @@ class JNIHandles : AllStatic {
 
   // Local handles
   static jobject make_local(oop obj);
-  static jobject make_local(Thread* thread, oop obj); // Faster version when current thread is known
+  static jobject make_local(Thread* thread, oop obj,  // Faster version when current thread is known
+                            AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
   inline static void destroy_local(jobject handle);
 
   // Global handles
-  static jobject make_global(Handle  obj, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
+  static jobject make_global(Handle  obj,
+                             AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
   static void destroy_global(jobject handle);
 
   // Weak global handles
-  static jobject make_weak_global(Handle obj, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
+  static jobject make_weak_global(Handle obj,
+                                  AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
   static void destroy_weak_global(jobject handle);
   static bool is_global_weak_cleared(jweak handle); // Test jweak without resolution
 
@@ -176,10 +179,10 @@ class JNIHandleBlock : public CHeapObj<mtInternal> {
 
  public:
   // Handle allocation
-  jobject allocate_handle(oop obj);
+  jobject allocate_handle(oop obj, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
 
   // Block allocation and block free list management
-  static JNIHandleBlock* allocate_block(Thread* thread = NULL);
+  static JNIHandleBlock* allocate_block(Thread* thread = NULL, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
   static void release_block(JNIHandleBlock* block, Thread* thread = NULL);
 
   // JNI PushLocalFrame/PopLocalFrame support