8320890: [AIX] Find a better way to mimic dl handle equality

Backport-of: b8ae4a8c0985d1763ac48ba78943d8b992d7be77

src/hotspot/os/aix/os_aix.cpp

@@ -1118,7 +1118,9 @@ void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
   }
 
   if (!filename || strlen(filename) == 0) {
-    ::strncpy(ebuf, "dll_load: empty filename specified", ebuflen - 1);
+    if (ebuf != nullptr && ebuflen > 0) {
+      ::strncpy(ebuf, "dll_load: empty filename specified", ebuflen - 1);
+    }
     return nullptr;
   }
 
@@ -1133,8 +1135,9 @@ void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
   }
 
   void* result;
+  const char* error_report = nullptr;
   JFR_ONLY(NativeLibraryLoadEvent load_event(filename, &result);)
-  result = ::dlopen(filename, dflags);
+  result = Aix_dlopen(filename, dflags, &error_report);
   if (result != nullptr) {
     Events::log_dll_message(nullptr, "Loaded shared library %s", filename);
     // Reload dll cache. Don't do this in signal handling.
@@ -1143,7 +1146,6 @@ void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
     return result;
   } else {
     // error analysis when dlopen fails
-    const char* error_report = ::dlerror();
     if (error_report == nullptr) {
       error_report = "dlerror returned no error description";
     }
@@ -3031,31 +3033,3 @@ void os::jfr_report_memory_info() {}
 
 #endif // INCLUDE_JFR
 
-// Simulate the library search algorithm of dlopen() (in os::dll_load)
-int os::Aix::stat64x_via_LIBPATH(const char* path, struct stat64x* stat) {
-  if (path[0] == '/' ||
-      (path[0] == '.' && (path[1] == '/' ||
-                          (path[1] == '.' && path[2] == '/')))) {
-    return stat64x(path, stat);
-  }
-
-  const char* env = getenv("LIBPATH");
-  if (env == nullptr || *env == 0)
-    return -1;
-
-  int ret = -1;
-  size_t libpathlen = strlen(env);
-  char* libpath = NEW_C_HEAP_ARRAY(char, libpathlen + 1, mtServiceability);
-  char* combined = NEW_C_HEAP_ARRAY(char, libpathlen + strlen(path) + 1, mtServiceability);
-  char *saveptr, *token;
-  strcpy(libpath, env);
-  for (token = strtok_r(libpath, ":", &saveptr); token != nullptr; token = strtok_r(nullptr, ":", &saveptr)) {
-    sprintf(combined, "%s/%s", token, path);
-    if (0 == (ret = stat64x(combined, stat)))
-      break;
-  }
-
-  FREE_C_HEAP_ARRAY(char*, combined);
-  FREE_C_HEAP_ARRAY(char*, libpath);
-  return ret;
-}

src/hotspot/os/aix/os_aix.hpp

@@ -175,8 +175,6 @@ class os::Aix {
   static bool platform_print_native_stack(outputStream* st, const void* context, char *buf, int buf_size, address& lastpc);
   static void* resolve_function_descriptor(void* p);
 
-  // Simulate the library search algorithm of dlopen() (in os::dll_load)
-  static int stat64x_via_LIBPATH(const char* path, struct stat64x* stat);
 };
 
 #endif // OS_AIX_OS_AIX_HPP

src/hotspot/os/aix/porting_aix.cpp

@@ -21,6 +21,12 @@
  * questions.
  *
  */
+// needs to be defined first, so that the implicit loaded xcoff.h header defines
+// the right structures to analyze the loader header of 64 Bit executable files
+// this is needed for rtv_linkedin_libpath() to get the linked (burned) in library
+// search path of an XCOFF executable
+#define __XCOFF64__
+#include <xcoff.h>
 
 #include "asm/assembler.hpp"
 #include "compiler/disassembler.hpp"
@@ -891,3 +897,275 @@ bool AixMisc::query_stack_bounds_for_current_thread(stackbounds_t* out) {
   return true;
 
 }
+
+// variables needed to emulate linux behavior in os::dll_load() if library is loaded twice
+static pthread_mutex_t g_handletable_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+struct TableLocker {
+  TableLocker() { pthread_mutex_lock(&g_handletable_mutex); }
+  ~TableLocker() { pthread_mutex_unlock(&g_handletable_mutex); }
+};
+struct handletableentry{
+    void*   handle;
+    ino64_t inode;
+    dev64_t devid;
+    uint    refcount;
+};
+constexpr unsigned init_num_handles = 128;
+static unsigned max_handletable = 0;
+static unsigned g_handletable_used = 0;
+// We start with an empty array. At first use we will dynamically allocate memory for 128 entries.
+// If this table is full we dynamically reallocate a memory reagion of double size, and so on.
+static struct handletableentry* p_handletable = nullptr;
+
+// get the library search path burned in to the executable file during linking
+// If the libpath cannot be retrieved return an empty path
+static const char* rtv_linkedin_libpath() {
+  constexpr int bufsize = 4096;
+  static char buffer[bufsize];
+  static const char* libpath = 0;
+
+  // we only try to retrieve the libpath once. After that try we
+  // let libpath point to buffer, which then contains a valid libpath
+  // or an empty string
+  if (libpath != nullptr) {
+    return libpath;
+  }
+
+  // retrieve the path to the currently running executable binary
+  // to open it
+  snprintf(buffer, 100, "/proc/%ld/object/a.out", (long)getpid());
+  FILE* f = nullptr;
+  struct xcoffhdr the_xcoff;
+  struct scnhdr the_scn;
+  struct ldhdr the_ldr;
+  constexpr size_t xcoffsz = FILHSZ + _AOUTHSZ_EXEC;
+  STATIC_ASSERT(sizeof(the_xcoff) == xcoffsz);
+  STATIC_ASSERT(sizeof(the_scn) == SCNHSZ);
+  STATIC_ASSERT(sizeof(the_ldr) == LDHDRSZ);
+  // read the generic XCOFF header and analyze the substructures
+  // to find the burned in libpath. In any case of error perform the assert
+  if (nullptr == (f = fopen(buffer, "r")) ||
+      xcoffsz != fread(&the_xcoff, 1, xcoffsz, f) ||
+      the_xcoff.filehdr.f_magic != U64_TOCMAGIC ||
+      0 != fseek(f, (FILHSZ + the_xcoff.filehdr.f_opthdr + (the_xcoff.aouthdr.o_snloader -1)*SCNHSZ), SEEK_SET) ||
+      SCNHSZ != fread(&the_scn, 1, SCNHSZ, f) ||
+      0 != strcmp(the_scn.s_name, ".loader") ||
+      0 != fseek(f, the_scn.s_scnptr, SEEK_SET) ||
+      LDHDRSZ != fread(&the_ldr, 1, LDHDRSZ, f) ||
+      0 != fseek(f, the_scn.s_scnptr + the_ldr.l_impoff, SEEK_SET) ||
+      0 == fread(buffer, 1, bufsize, f)) {
+    buffer[0] = 0;
+    assert(false, "could not retrieve burned in library path from executables loader section");
+  }
+
+  if (f) {
+    fclose(f);
+  }
+  libpath = buffer;
+
+  return libpath;
+}
+
+// Simulate the library search algorithm of dlopen() (in os::dll_load)
+static bool search_file_in_LIBPATH(const char* path, struct stat64x* stat) {
+  if (path == nullptr)
+    return false;
+
+  char* path2 = os::strdup(path);
+  // if exist, strip off trailing (shr_64.o) or similar
+  char* substr;
+  if (path2[strlen(path2) - 1] == ')' && (substr = strrchr(path2, '('))) {
+    *substr = 0;
+  }
+
+  bool ret = false;
+  // If FilePath contains a slash character, FilePath is used directly,
+  // and no directories are searched.
+  // But if FilePath does not start with / or . we have to prepend it with ./
+  if (strchr(path2, '/')) {
+    stringStream combined;
+    if (*path2 == '/' || *path2 == '.') {
+      combined.print("%s", path2);
+    } else {
+      combined.print("./%s", path2);
+    }
+    ret = (0 == stat64x(combined.base(), stat));
+    os::free(path2);
+    return ret;
+  }
+
+  const char* env = getenv("LIBPATH");
+  if (env == nullptr) {
+    // no LIBPATH, try with LD_LIBRARY_PATH
+    env = getenv("LD_LIBRARY_PATH");
+  }
+
+  stringStream Libpath;
+  if (env == nullptr) {
+    // no LIBPATH or LD_LIBRARY_PATH given -> try only with burned in libpath
+    Libpath.print("%s", rtv_linkedin_libpath());
+  } else if (*env == 0) {
+    // LIBPATH or LD_LIBRARY_PATH given but empty -> try first with burned
+    //  in libpath and with current working directory second
+    Libpath.print("%s:.", rtv_linkedin_libpath());
+  } else {
+    // LIBPATH or LD_LIBRARY_PATH given with content -> try first with
+    // LIBPATH or LD_LIBRARY_PATH and second with burned in libpath.
+    // No check against current working directory
+    Libpath.print("%s:%s", env, rtv_linkedin_libpath());
+  }
+
+  char* libpath = os::strdup(Libpath.base());
+
+  char *saveptr, *token;
+  for (token = strtok_r(libpath, ":", &saveptr); token != nullptr; token = strtok_r(nullptr, ":", &saveptr)) {
+    stringStream combined;
+    combined.print("%s/%s", token, path2);
+    if ((ret = (0 == stat64x(combined.base(), stat))))
+      break;
+  }
+
+  os::free(libpath);
+  os::free(path2);
+  return ret;
+}
+
+// specific AIX versions for ::dlopen() and ::dlclose(), which handles the struct g_handletable
+// This way we mimic dl handle equality for a library
+// opened a second time, as it is implemented on other platforms.
+void* Aix_dlopen(const char* filename, int Flags, const char** error_report) {
+  assert(error_report != nullptr, "error_report is nullptr");
+  void* result;
+  struct stat64x libstat;
+
+  if (false == search_file_in_LIBPATH(filename, &libstat)) {
+    // file with filename does not exist
+  #ifdef ASSERT
+    result = ::dlopen(filename, Flags);
+    assert(result == nullptr, "dll_load: Could not stat() file %s, but dlopen() worked; Have to improve stat()", filename);
+  #endif
+    *error_report = "Could not load module .\nSystem error: No such file or directory";
+    return nullptr;
+  }
+  else {
+    unsigned i = 0;
+    TableLocker lock;
+    // check if library belonging to filename is already loaded.
+    // If yes use stored handle from previous ::dlopen() and increase refcount
+    for (i = 0; i < g_handletable_used; i++) {
+      if ((p_handletable + i)->handle &&
+          (p_handletable + i)->inode == libstat.st_ino &&
+          (p_handletable + i)->devid == libstat.st_dev) {
+        (p_handletable + i)->refcount++;
+        result = (p_handletable + i)->handle;
+        break;
+      }
+    }
+    if (i == g_handletable_used) {
+      // library not yet loaded. Check if there is space left in array
+      // to store new ::dlopen() handle
+      if (g_handletable_used == max_handletable) {
+        // No place in array anymore; increase array.
+        unsigned new_max = MAX2(max_handletable * 2, init_num_handles);
+        struct handletableentry* new_tab = (struct handletableentry*)::realloc(p_handletable, new_max * sizeof(struct handletableentry));
+        assert(new_tab != nullptr, "no more memory for handletable");
+        if (new_tab == nullptr) {
+          *error_report = "dlopen: no more memory for handletable";
+          return nullptr;
+        }
+        max_handletable = new_max;
+        p_handletable = new_tab;
+      }
+      // Library not yet loaded; load it, then store its handle in handle table
+      result = ::dlopen(filename, Flags);
+      if (result != nullptr) {
+        g_handletable_used++;
+        (p_handletable + i)->handle = result;
+        (p_handletable + i)->inode = libstat.st_ino;
+        (p_handletable + i)->devid = libstat.st_dev;
+        (p_handletable + i)->refcount = 1;
+      }
+      else {
+        // error analysis when dlopen fails
+        *error_report = ::dlerror();
+        if (*error_report == nullptr) {
+          *error_report = "dlerror returned no error description";
+        }
+      }
+    }
+  }
+  return result;
+}
+
+bool os::pd_dll_unload(void* libhandle, char* ebuf, int ebuflen) {
+  unsigned i = 0;
+  bool res = false;
+
+  if (ebuf && ebuflen > 0) {
+    ebuf[0] = '\0';
+    ebuf[ebuflen - 1] = '\0';
+  }
+
+  {
+    TableLocker lock;
+    // try to find handle in array, which means library was loaded by os::dll_load() call
+    for (i = 0; i < g_handletable_used; i++) {
+      if ((p_handletable + i)->handle == libhandle) {
+        // handle found, decrease refcount
+        assert((p_handletable + i)->refcount > 0, "Sanity");
+        (p_handletable + i)->refcount--;
+        if ((p_handletable + i)->refcount > 0) {
+          // if refcount is still >0 then we have to keep library and just return true
+          return true;
+        }
+        // refcount == 0, so we have to ::dlclose() the lib
+        // and delete the entry from the array.
+        break;
+      }
+    }
+
+    // If we reach this point either the libhandle was found with refcount == 0, or the libhandle
+    // was not found in the array at all. In both cases we have to ::dlclose the lib and perform
+    // the error handling. In the first case we then also have to delete the entry from the array
+    // while in the second case we simply have to nag.
+    res = (0 == ::dlclose(libhandle));
+    if (!res) {
+      // error analysis when dlopen fails
+      const char* error_report = ::dlerror();
+      if (error_report == nullptr) {
+        error_report = "dlerror returned no error description";
+      }
+      if (ebuf != nullptr && ebuflen > 0) {
+        snprintf(ebuf, ebuflen - 1, "%s", error_report);
+      }
+      assert(false, "os::pd_dll_unload() ::dlclose() failed");
+    }
+
+    if (i < g_handletable_used) {
+      if (res) {
+        // First case: libhandle was found (with refcount == 0) and ::dlclose successful,
+        // so delete entry from array
+        g_handletable_used--;
+        // If the entry was the last one of the array, the previous g_handletable_used--
+        // is sufficient to remove the entry from the array, otherwise we move the last
+        // entry of the array to the place of the entry we want to remove and overwrite it
+        if (i < g_handletable_used) {
+          *(p_handletable + i) = *(p_handletable + g_handletable_used);
+          (p_handletable + g_handletable_used)->handle = nullptr;
+        }
+      }
+    }
+    else {
+      // Second case: libhandle was not found (library was not loaded by os::dll_load())
+      // therefore nag
+      assert(false, "os::pd_dll_unload() library was not loaded by os::dll_load()");
+    }
+  }
+
+  // Update the dll cache
+  LoadedLibraries::reload();
+
+  return res;
+} // end: os::pd_dll_unload()
+

src/hotspot/os/aix/porting_aix.hpp

@@ -115,4 +115,6 @@ class AixMisc {
 
 };
 
+void* Aix_dlopen(const char* filename, int Flags, const char** error_report);
+
 #endif // OS_AIX_PORTING_AIX_HPP

src/hotspot/os/bsd/os_bsd.cpp

@@ -2557,3 +2557,25 @@ void os::jfr_report_memory_info() {
 }
 
 #endif // INCLUDE_JFR
+
+bool os::pd_dll_unload(void* libhandle, char* ebuf, int ebuflen) {
+
+  if (ebuf && ebuflen > 0) {
+    ebuf[0] = '\0';
+    ebuf[ebuflen - 1] = '\0';
+  }
+
+  bool res = (0 == ::dlclose(libhandle));
+  if (!res) {
+    // error analysis when dlopen fails
+    const char* error_report = ::dlerror();
+    if (error_report == nullptr) {
+      error_report = "dlerror returned no error description";
+    }
+    if (ebuf != nullptr && ebuflen > 0) {
+      snprintf(ebuf, ebuflen - 1, "%s", error_report);
+    }
+  }
+
+  return res;
+} // end: os::pd_dll_unload()