[Libomptarget] Replace use of dlopen with LLVM's dynamic library su…

…pport

This patch replaces uses of `dlopen` and `dlsym` with LLVM's support
with `loadPermanentLibrary` and `getSymbolAddress`. This allows us to
remove the explicit dependency on the `dl` libraries in the CMake. This
removes another explicit dependency and solves an issue encountered
while building on Windows platforms. The one downside to this is that
the LLVM library does not currently support `dlclose` functionality, but
this could be added in the future.

Reviewed By: JonChesterfield

Differential Revision: https://reviews.llvm.org/D131507

openmp/libomptarget/include/rtl.h

@@ -13,8 +13,10 @@
 #ifndef _OMPTARGET_RTL_H
 #define _OMPTARGET_RTL_H
 
-#include "omptarget.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/DynamicLibrary.h"
+
+#include "omptarget.h"
 
 #include <list>
 #include <map>
@@ -78,7 +80,7 @@ struct RTLInfoTy {
                                 // to be registered with this RTL.
   int32_t NumberOfDevices = -1; // Number of devices this RTL deals with.
 
-  void *LibraryHandler = nullptr;
+  std::unique_ptr<llvm::sys::DynamicLibrary> LibraryHandler;
 
 #ifdef OMPTARGET_DEBUG
   std::string RTLName;

openmp/libomptarget/plugins/CMakeLists.txt

@@ -45,7 +45,6 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$")
         elf_common
         ${LIBOMPTARGET_DEP_LIBFFI_LIBRARIES}
         ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
-        dl
         ${OPENMP_PTHREAD_LIB}
         "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"
 

openmp/libomptarget/plugins/amdgpu/CMakeLists.txt

@@ -84,7 +84,6 @@ add_llvm_library(omptarget.rtl.amdgpu SHARED
   PRIVATE
   elf_common
   ${LIBOMPTARGET_DEP_LIBRARIES}
-  ${CMAKE_DL_LIBS}
   ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
   ${OPENMP_PTHREAD_LIB}
   "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"

openmp/libomptarget/plugins/amdgpu/dynamic_hsa/hsa.cpp

@@ -10,12 +10,14 @@
 // Does the dlopen/dlsym calls as part of the call to hsa_init
 //
 //===----------------------------------------------------------------------===//
-#include "hsa.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
 #include "Debug.h"
 #include "dlwrap.h"
+#include "hsa.h"
 #include "hsa_ext_amd.h"
-
-#include <dlfcn.h>
+#include <memory>
 
 DLWRAP_INITIALIZE();
 
@@ -72,16 +74,18 @@ static bool checkForHSA() {
   // return true if dlopen succeeded and all functions found
 
   const char *HsaLib = DYNAMIC_HSA_PATH;
-  void *DynlibHandle = dlopen(HsaLib, RTLD_NOW);
-  if (!DynlibHandle) {
-    DP("Unable to load library '%s': %s!\n", HsaLib, dlerror());
+  std::string ErrMsg;
+  auto DynlibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
+      llvm::sys::DynamicLibrary::getPermanentLibrary(HsaLib, &ErrMsg));
+  if (!DynlibHandle->isValid()) {
+    DP("Unable to load library '%s': %s!\n", HsaLib, ErrMsg.c_str());
     return false;
   }
 
   for (size_t I = 0; I < dlwrap::size(); I++) {
     const char *Sym = dlwrap::symbol(I);
 
-    void *P = dlsym(DynlibHandle, Sym);
+    void *P = DynlibHandle->getAddressOfSymbol(Sym);
     if (P == nullptr) {
       DP("Unable to find '%s' in '%s'!\n", Sym, HsaLib);
       return false;

openmp/libomptarget/plugins/cuda/CMakeLists.txt

@@ -77,7 +77,6 @@ else()
     PRIVATE
     elf_common
     MemoryManager
-    ${CMAKE_DL_LIBS}
     ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
     ${OPENMP_PTHREAD_LIB}
     "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"

openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp

@@ -11,15 +11,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "cuda.h"
+#include "llvm/Support/DynamicLibrary.h"
+
 #include "Debug.h"
+#include "cuda.h"
 #include "dlwrap.h"
 
+#include <memory>
 #include <string>
 #include <unordered_map>
 
-#include <dlfcn.h>
-
 DLWRAP_INITIALIZE();
 
 DLWRAP_INTERNAL(cuInit, 1);
@@ -105,9 +106,11 @@ static bool checkForCUDA() {
   };
 
   const char *CudaLib = DYNAMIC_CUDA_PATH;
-  void *DynlibHandle = dlopen(CudaLib, RTLD_NOW);
-  if (!DynlibHandle) {
-    DP("Unable to load library '%s': %s!\n", CudaLib, dlerror());
+  std::string ErrMsg;
+  auto DynlibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
+      llvm::sys::DynamicLibrary::getPermanentLibrary(CudaLib, &ErrMsg));
+  if (!DynlibHandle->isValid()) {
+    DP("Unable to load library '%s': %s!\n", CudaLib, ErrMsg.c_str());
     return false;
   }
 
@@ -117,15 +120,15 @@ static bool checkForCUDA() {
     auto It = TryFirst.find(Sym);
     if (It != TryFirst.end()) {
       const char *First = It->second;
-      void *P = dlsym(DynlibHandle, First);
+      void *P = DynlibHandle->getAddressOfSymbol(First);
       if (P) {
         DP("Implementing %s with dlsym(%s) -> %p\n", Sym, First, P);
         *dlwrap::pointer(I) = P;
         continue;
       }
     }
 
-    void *P = dlsym(DynlibHandle, Sym);
+    void *P = DynlibHandle->getAddressOfSymbol(Sym);
     if (P == nullptr) {
       DP("Unable to find '%s' in '%s'!\n", Sym, CudaLib);
       return false;

openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp

@@ -11,12 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/DynamicLibrary.h"
 
 #include <cassert>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
-#include <dlfcn.h>
 #include <ffi.h>
 #include <gelf.h>
 #include <link.h>
@@ -28,6 +28,7 @@
 #include "omptargetplugin.h"
 
 using namespace llvm;
+using namespace llvm::sys;
 
 #ifndef TARGET_NAME
 #define TARGET_NAME Generic ELF - 64bit
@@ -46,7 +47,7 @@ using namespace llvm;
 /// Array of Dynamic libraries loaded for this target.
 struct DynLibTy {
   std::string FileName;
-  void *Handle;
+  std::unique_ptr<DynamicLibrary> DynLib;
 };
 
 /// Keep entries table per device.
@@ -105,10 +106,8 @@ class RTLDeviceInfoTy {
   ~RTLDeviceInfoTy() {
     // Close dynamic libraries
     for (auto &Lib : DynLibs) {
-      if (Lib.Handle) {
-        dlclose(Lib.Handle);
+      if (Lib.DynLib->isValid())
         remove(Lib.FileName.c_str());
-      }
     }
   }
 };
@@ -164,15 +163,16 @@ __tgt_target_table *__tgt_rtl_load_binary(int32_t DeviceId,
   fwrite(Image->ImageStart, ImageSize, 1, Ftmp);
   fclose(Ftmp);
 
-  DynLibTy Lib = {TmpName, dlopen(TmpName, RTLD_LAZY)};
+  std::string ErrMsg;
+  auto DynLib = std::make_unique<sys::DynamicLibrary>(
+      sys::DynamicLibrary::getPermanentLibrary(TmpName, &ErrMsg));
+  DynLibTy Lib = {TmpName, std::move(DynLib)};
 
-  if (!Lib.Handle) {
-    DP("Target library loading error: %s\n", dlerror());
+  if (!Lib.DynLib->isValid()) {
+    DP("Target library loading error: %s\n", ErrMsg.c_str());
     return NULL;
   }
 
-  DeviceInfo.DynLibs.push_back(Lib);
-
   __tgt_offload_entry *HostBegin = Image->EntriesBegin;
   __tgt_offload_entry *HostEnd = Image->EntriesEnd;
 
@@ -184,7 +184,7 @@ __tgt_target_table *__tgt_rtl_load_binary(int32_t DeviceId,
 
     __tgt_offload_entry Entry = *E;
 
-    void *DevAddr = dlsym(Lib.Handle, E->name);
+    void *DevAddr = Lib.DynLib->getAddressOfSymbol(E->name);
     Entry.addr = DevAddr;
 
     DP("Entry point " DPxMOD " maps to global %s (" DPxMOD ")\n",
@@ -194,6 +194,7 @@ __tgt_target_table *__tgt_rtl_load_binary(int32_t DeviceId,
   }
 
   DeviceInfo.createOffloadTable(DeviceId, std::move(Entries));
+  DeviceInfo.DynLibs.emplace_back(std::move(Lib));
 
   return DeviceInfo.getOffloadEntriesTable(DeviceId);
 }

openmp/libomptarget/src/CMakeLists.txt

@@ -32,7 +32,6 @@ add_llvm_library(omptarget
 
   LINK_LIBS 
   PRIVATE
-  ${CMAKE_DL_LIBS}
   "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports"
   NO_INSTALL_RPATH
 )