8255784: appcds/javaldr/ExceptionDuringDumpAtObjectsInitPhase.java test failed resulting in VM crash

Reviewed-by: ccheung, iklam

Author: yminqi

src/hotspot/share/memory/archiveUtils.cpp

@@ -323,8 +323,8 @@ void ArchiveUtils::log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) {
 void ArchiveUtils::check_for_oom(oop exception) {
   assert(exception != nullptr, "Sanity check");
   if (exception->is_a(SystemDictionary::OutOfMemoryError_klass())) {
-    vm_exit_during_cds_dumping(
-      err_msg("Out of memory. Please run with a larger Java heap, current MaxHeapSize = " SIZE_FORMAT "M",
-              MaxHeapSize/M));
+    vm_direct_exit(-1,
+      err_msg("Out of memory. Please run with a larger Java heap, current MaxHeapSize = "
+              SIZE_FORMAT "M", MaxHeapSize/M));
   }
 }

src/hotspot/share/memory/heapShared.cpp

@@ -206,7 +206,9 @@ oop HeapShared::archive_heap_object(oop obj, Thread* THREAD) {
     log_error(cds, heap)(
       "Cannot allocate space for object " PTR_FORMAT " in archived heap region",
       p2i(obj));
-    vm_exit(1);
+    vm_direct_exit(-1,
+      err_msg("Out of memory. Please run with a larger Java heap, current MaxHeapSize = "
+              SIZE_FORMAT "M", MaxHeapSize/M));
   }
   return archived_oop;
 }
@@ -725,7 +727,7 @@ oop HeapShared::archive_reachable_objects_from(int level,
     // these objects that are referenced (directly or indirectly) by static fields.
     ResourceMark rm;
     log_error(cds, heap)("Cannot archive object of class %s", orig_obj->klass()->external_name());
-    vm_exit(1);
+    vm_direct_exit(1);
   }
 
   // java.lang.Class instances cannot be included in an archived object sub-graph. We only support
@@ -735,7 +737,7 @@ oop HeapShared::archive_reachable_objects_from(int level,
   // object that is referenced (directly or indirectly) by static fields.
   if (java_lang_Class::is_instance(orig_obj)) {
     log_error(cds, heap)("(%d) Unknown java.lang.Class object is in the archived sub-graph", level);
-    vm_exit(1);
+    vm_direct_exit(1);
   }
 
   oop archived_obj = find_archived_heap_object(orig_obj);
@@ -771,7 +773,7 @@ oop HeapShared::archive_reachable_objects_from(int level,
         // We don't know how to handle an object that has been archived, but some of its reachable
         // objects cannot be archived. Bail out for now. We might need to fix this in the future if
         // we have a real use case.
-        vm_exit(1);
+        vm_direct_exit(1);
       }
     }
 
@@ -1031,7 +1033,7 @@ void HeapShared::init_subgraph_entry_fields(ArchivableStaticFieldInfo fields[],
       ArchiveUtils::check_for_oom(PENDING_EXCEPTION); // exit on OOM
       log_info(cds)("%s: %s", PENDING_EXCEPTION->klass()->external_name(),
                     java_lang_String::as_utf8_string(java_lang_Throwable::message(PENDING_EXCEPTION)));
-      vm_exit_during_initialization("VM exits due to exception, use -Xlog:cds,exceptions=trace for detail");
+      vm_direct_exit(-1, "VM exits due to exception, use -Xlog:cds,exceptions=trace for detail");
     }
     InstanceKlass* ik = InstanceKlass::cast(k);
     assert(InstanceKlass::cast(ik)->is_shared_boot_class(),

src/hotspot/share/runtime/java.cpp

@@ -578,6 +578,13 @@ void vm_direct_exit(int code) {
   os::exit(code);
 }
 
+void vm_direct_exit(int code, const char* message) {
+  if (message != nullptr) {
+    tty->print_cr("%s", message);
+  }
+  vm_direct_exit(code);
+}
+
 void vm_perform_shutdown_actions() {
   if (is_init_completed()) {
     Thread* thread = Thread::current_or_null();

src/hotspot/share/runtime/java.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,6 +35,7 @@ extern void vm_exit(int code);
 
 // Wrapper for ::exit()
 extern void vm_direct_exit(int code);
+extern void vm_direct_exit(int code, const char* message);
 
 // Shutdown the VM but do not exit the process
 extern void vm_shutdown();