8295889: NMT preinit code does not handle allocation errors

Reviewed-by: dholmes, mbaesken

Author: tstuefe

src/hotspot/share/services/nmtPreInit.cpp

@@ -41,11 +41,22 @@ static void  raw_free(void* p)                  { ALLOW_C_FUNCTION(::free, ::fre
 static const size_t malloc_alignment = 2 * sizeof(void*); // could we use max_align_t?
 STATIC_ASSERT(is_aligned(sizeof(NMTPreInitAllocation), malloc_alignment));
 
+// To keep matters simple we just raise a fatal error on OOM. Since preinit allocation
+// is just used for pre-VM-initialization mallocs, none of which are optional, we don't
+// need a finer grained error handling.
+static void fail_oom(size_t size) {
+  vm_exit_out_of_memory(size, OOM_MALLOC_ERROR, "VM early initialization phase");
+}
+
 // --------- NMTPreInitAllocation --------------
 
 NMTPreInitAllocation* NMTPreInitAllocation::do_alloc(size_t payload_size) {
   const size_t outer_size = sizeof(NMTPreInitAllocation) + payload_size;
+  guarantee(outer_size > payload_size, "Overflow");
   void* p = raw_malloc(outer_size);
+  if (p == nullptr) {
+    fail_oom(outer_size);
+  }
   NMTPreInitAllocation* a = new(p) NMTPreInitAllocation(payload_size);
   return a;
 }
@@ -54,7 +65,11 @@ NMTPreInitAllocation* NMTPreInitAllocation::do_reallocate(NMTPreInitAllocation*
   assert(old->next == NULL, "unhang from map first");
   // We just reallocate the old block, header and all.
   const size_t new_outer_size = sizeof(NMTPreInitAllocation) + new_payload_size;
+  guarantee(new_outer_size > new_payload_size, "Overflow");
   void* p = raw_realloc(old, new_outer_size);
+  if (p == nullptr) {
+    fail_oom(new_outer_size);
+  }
   // re-stamp header with new size
   NMTPreInitAllocation* a = new(p) NMTPreInitAllocation(new_payload_size);
   return a;

test/hotspot/gtest/nmt/test_nmtpreinit.cpp

@@ -90,6 +90,25 @@ class TestAllocations {
     os::free(NULL);                      // free(null)
     DEBUG_ONLY(NMTPreInit::verify();)
 
+    // This should result in a fatal native oom error from NMT preinit with a clear error message.
+    // It should not result in a SEGV or anything similar. Unfortunately difficult to test
+    // automatically.
+    // Uncomment to test manually.
+
+    // case 1: overflow
+    // os_malloc(SIZE_MAX);
+
+    // case 2: failing malloc
+    // os_malloc(SIZE_MAX - M);
+
+    // case 3: overflow in realloc
+    // void* p = os_malloc(10);
+    // p = os_realloc(p, SIZE_MAX);
+
+    // case 4: failing realloc
+    // void* p = os_malloc(10);
+    // p = os_realloc(p, SIZE_MAX - M);
+
     log_state();
   }
   void test_post() {