8362846: Windows error reporting for dll_load doesn't check for a null buffer

8362954: Missing error buffer null check in os::dll_load on Linux/BSD

Reviewed-by: mgronlun, kbarrett

Author: David Holmes

src/hotspot/os/bsd/os_bsd.cpp

@@ -1110,7 +1110,10 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
   if (result != nullptr) {
     return result;
   }
-
+  if (ebuf == nullptr || ebuflen < 1) {
+    // no error reporting requested
+    return nullptr;
+  }
   Events::log_dll_message(nullptr, "Loading shared library %s failed, %s", filename, error_report);
   log_info(os)("shared library load of %s failed, %s", filename, error_report);
   int diag_msg_max_length=ebuflen-strlen(ebuf);

src/hotspot/os/linux/os_linux.cpp

@@ -1685,6 +1685,11 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
     return result;
   }
 
+  if (ebuf == nullptr || ebuflen < 1) {
+    // no error reporting requested
+    return nullptr;
+  }
+
   Elf32_Ehdr elf_head;
   size_t prefix_len = strlen(ebuf);
   ssize_t diag_msg_max_length = ebuflen - prefix_len;

src/hotspot/os/windows/os_windows.cpp

@@ -1729,6 +1729,12 @@ void * os::dll_load(const char *name, char *ebuf, int ebuflen) {
     log_info(os)("shared library load of %s was successful", name);
     return result;
   }
+
+  if (ebuf == nullptr || ebuflen < 1) {
+    // no error reporting requested
+    return nullptr;
+  }
+
   DWORD errcode = GetLastError();
   // Read system error message into ebuf
   // It may or may not be overwritten below (in the for loop and just above)
@@ -2261,6 +2267,8 @@ void os::jvm_path(char *buf, jint buflen) {
 // from src/windows/hpi/src/system_md.c
 
 size_t os::lasterror(char* buf, size_t len) {
+  assert(buf != nullptr && len > 0, "invalid buffer passed");
+
   DWORD errval;
 
   if ((errval = GetLastError()) != 0) {

test/hotspot/gtest/runtime/test_os.cpp

@@ -1181,3 +1181,9 @@ TEST_VM(os, map_memory_to_file_aligned) {
 }
 
 #endif // !defined(_AIX)
+
+TEST_VM(os, dll_load_null_error_buf) {
+  // This should not crash.
+  void* lib = os::dll_load("NoSuchLib", nullptr, 0);
+  ASSERT_NULL(lib);
+}