Skip to content

Commit

Permalink
[Libomptarget] Allow the CPU targets to be built without libffi (llvm…
Browse files Browse the repository at this point in the history
…#77495)

Summary:
The CPU targets currently rely on `libffi` to invoke the "kernel"
functions. Previously we would not build these if this dependency was
not found. This patch copies th eapproach used for things like CUDA and
HSA to dynamically load this if it is not found.

The one sketchy thing this does is hard-code the default ABI for the
target. These are normally defined on a per-file basis in the FFI
source, so I had to fish out the expected values. We only use two types,
so ideally we will always be able to use the default ABI.

It's possible we could remove this dependency entirely in the future as
well.
  • Loading branch information
jhuber6 committed Jan 9, 2024
1 parent b6d1577 commit c7c68f1
Show file tree
Hide file tree
Showing 5 changed files with 209 additions and 102 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -50,52 +50,8 @@ endif()
################################################################################
# Looking for libffi...
################################################################################
find_package(PkgConfig)

pkg_check_modules(LIBOMPTARGET_SEARCH_LIBFFI QUIET libffi)

find_path (
LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIR
NAMES
ffi.h
HINTS
${LIBOMPTARGET_SEARCH_LIBFFI_INCLUDEDIR}
${LIBOMPTARGET_SEARCH_LIBFFI_INCLUDE_DIRS}
PATHS
/usr/include
/usr/local/include
/opt/local/include
/sw/include
ENV CPATH)

# Don't bother look for the library if the header files were not found.
if (LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIR)
find_library (
LIBOMPTARGET_DEP_LIBFFI_LIBRARIES
NAMES
ffi
HINTS
${LIBOMPTARGET_SEARCH_LIBFFI_LIBDIR}
${LIBOMPTARGET_SEARCH_LIBFFI_LIBRARY_DIRS}
PATHS
/usr/lib
/usr/local/lib
/opt/local/lib
/sw/lib
ENV LIBRARY_PATH
ENV LD_LIBRARY_PATH)
endif()

set(LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIRS ${LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIR})
find_package_handle_standard_args(
LIBOMPTARGET_DEP_LIBFFI
DEFAULT_MSG
LIBOMPTARGET_DEP_LIBFFI_LIBRARIES
LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIRS)

mark_as_advanced(
LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIRS
LIBOMPTARGET_DEP_LIBFFI_LIBRARIES)
find_package(FFI QUIET)
set(LIBOMPTARGET_DEP_LIBFFI_FOUND ${FFI_FOUND})

################################################################################
# Looking for CUDA...
Expand Down
115 changes: 59 additions & 56 deletions openmp/libomptarget/plugins-nextgen/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,71 +19,74 @@ add_subdirectory(common)
# - tmachine_libname: machine name to be appended to the plugin library name.
macro(build_generic_elf64 tmachine tmachine_name tmachine_libname tmachine_triple elf_machine_id)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$")
if(LIBOMPTARGET_DEP_LIBFFI_FOUND)

libomptarget_say("Building ${tmachine_name} NextGen offloading plugin.")

# Define macro to be used as prefix of the runtime messages for this target.
add_definitions("-DTARGET_NAME=${tmachine_name}")
# Define macro to be used as prefix of the runtime messages for this target.
add_definitions("-DTARGET_NAME=${tmachine_name}")

# Define debug prefix. TODO: This should be automatized in the Debug.h but
# it requires changing the original plugins.
add_definitions(-DDEBUG_PREFIX="TARGET ${tmachine_name} RTL")
# Define debug prefix. TODO: This should be automatized in the Debug.h but
# it requires changing the original plugins.
add_definitions(-DDEBUG_PREFIX="TARGET ${tmachine_name} RTL")

# Define macro with the ELF ID for this target.
add_definitions("-DTARGET_ELF_ID=${elf_machine_id}")
# Define the macro with the ELF e_machine for this target.
add_definitions("-DTARGET_ELF_ID=${elf_machine_id}")

# Define target regiple
add_definitions("-DLIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE=${tmachine}")
# Define target triple
add_definitions("-DLIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE=${tmachine}")

add_llvm_library("omptarget.rtl.${tmachine_libname}"
SHARED
add_llvm_library("omptarget.rtl.${tmachine_libname}"
SHARED

${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/src/rtl.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/src/rtl.cpp

ADDITIONAL_HEADER_DIRS
ADDITIONAL_HEADER_DIRS
${LIBOMPTARGET_INCLUDE_DIR}
${LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIR}

LINK_LIBS
PRIVATE
PluginCommon
${LIBOMPTARGET_DEP_LIBFFI_LIBRARIES}
${OPENMP_PTHREAD_LIB}

NO_INSTALL_RPATH
)

if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT))
target_link_libraries("omptarget.rtl.${tmachine_libname}" PRIVATE OMPT)
endif()

if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
target_link_libraries("omptarget.rtl.${tmachine_libname}" PRIVATE
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
endif()

# Install plugin under the lib destination folder.
install(TARGETS "omptarget.rtl.${tmachine_libname}"
LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
set_target_properties("omptarget.rtl.${tmachine_libname}" PROPERTIES
INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
POSITION_INDEPENDENT_CODE ON
CXX_VISIBILITY_PRESET protected)

target_include_directories( "omptarget.rtl.${tmachine_libname}" PRIVATE
${LIBOMPTARGET_INCLUDE_DIR}
${LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIR})

list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.${tmachine_libname}")
set(LIBOMPTARGET_TESTED_PLUGINS
"${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
set(LIBOMPTARGET_SYSTEM_TARGETS
"${LIBOMPTARGET_SYSTEM_TARGETS} ${tmachine_triple} ${tmachine_triple}-LTO" PARENT_SCOPE)
LINK_LIBS
PRIVATE
PluginCommon
${OPENMP_PTHREAD_LIB}

NO_INSTALL_RPATH
)

else(LIBOMPTARGET_DEP_LIBFFI_FOUND)
libomptarget_say("Not building ${tmachine_name} NextGen offloading plugin: libffi dependency not found.")
endif(LIBOMPTARGET_DEP_LIBFFI_FOUND)
if(LIBOMPTARGET_DEP_LIBFFI_FOUND)
libomptarget_say("Building ${tmachine_libname} plugin linked with libffi")
target_link_libraries("omptarget.rtl.${tmachine_libname}" PRIVATE
${FFI_LIBRARIES})
target_include_directories("omptarget.rtl.${tmachine_libname}" PRIVATE
${FFI_INCLUDE_DIRS})
else()
libomptarget_say("Building ${tmachine_libname} plugie for dlopened libffi")
target_sources("omptarget.rtl.${tmachine_libname}" PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/dynamic_ffi/ffi.cpp)
target_include_directories("omptarget.rtl.${tmachine_libname}" PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/dynamic_ffi)
endif()

if(OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
target_link_libraries("omptarget.rtl.${tmachine_libname}" PRIVATE OMPT)
endif()

if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
target_link_libraries("omptarget.rtl.${tmachine_libname}" PRIVATE
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
endif()

# Install plugin under the lib destination folder.
install(TARGETS "omptarget.rtl.${tmachine_libname}"
LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
set_target_properties("omptarget.rtl.${tmachine_libname}" PROPERTIES
INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
POSITION_INDEPENDENT_CODE ON
CXX_VISIBILITY_PRESET protected)

target_include_directories("omptarget.rtl.${tmachine_libname}" PRIVATE
${LIBOMPTARGET_INCLUDE_DIR})

list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.${tmachine_libname}")
set(LIBOMPTARGET_TESTED_PLUGINS
"${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
set(LIBOMPTARGET_SYSTEM_TARGETS
"${LIBOMPTARGET_SYSTEM_TARGETS} ${tmachine_triple} ${tmachine_triple}-LTO" PARENT_SCOPE)
else()
libomptarget_say("Not building ${tmachine_name} NextGen offloading plugin: machine not found in the system.")
endif()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
//===--- generic-elf-64bit/dynamic_ffi/ffi.cpp -------------------- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Implement subset of the FFI api by calling into the FFI library via dlopen
//
//===----------------------------------------------------------------------===//

#include "llvm/Support/DynamicLibrary.h"
#include <memory>

#include "DLWrap.h"
#include "ffi.h"

DLWRAP_INITIALIZE()

DLWRAP(ffi_call, 4);
DLWRAP(ffi_prep_cif, 5);

DLWRAP_FINALIZE()

ffi_type ffi_type_void;
ffi_type ffi_type_pointer;

// Name of the FFI shared library.
constexpr const char *FFI_PATH = "libffi.so";

#define DYNAMIC_FFI_SUCCESS 0
#define DYNAMIC_FFI_FAIL 1

// Initializes the dynamic FFI wrapper.
uint32_t ffi_init() {
std::string ErrMsg;
auto DynlibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
llvm::sys::DynamicLibrary::getPermanentLibrary(FFI_PATH, &ErrMsg));
if (!DynlibHandle->isValid())
return DYNAMIC_FFI_FAIL;

for (size_t I = 0; I < dlwrap::size(); I++) {
const char *Sym = dlwrap::symbol(I);

void *P = DynlibHandle->getAddressOfSymbol(Sym);
if (P == nullptr)
return DYNAMIC_FFI_FAIL;

*dlwrap::pointer(I) = P;
}

#define DYNAMIC_INIT(SYMBOL) \
{ \
void *SymbolPtr = DynlibHandle->getAddressOfSymbol(#SYMBOL); \
if (!SymbolPtr) \
return DYNAMIC_FFI_FAIL; \
SYMBOL = *reinterpret_cast<decltype(SYMBOL) *>(SymbolPtr); \
}
DYNAMIC_INIT(ffi_type_void);
DYNAMIC_INIT(ffi_type_pointer);
#undef DYNAMIC_INIT

return DYNAMIC_FFI_SUCCESS;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
//===--- generic-elf-64bit/dynamic_ffi/ffi.cpp -------------------- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Provides a mirror to the parts of the FFI interface that the plugins require.
//
// libffi
// - Copyright (c) 2011, 2014, 2019, 2021, 2022 Anthony Green
// - Copyright (c) 1996-2003, 2007, 2008 Red Hat, Inc.
//
//===----------------------------------------------------------------------===//

#ifndef DYNAMIC_FFI_FFI_H
#define DYNAMIC_FFI_FFI_H

#include <stddef.h>
#include <stdint.h>

#define USES_DYNAMIC_FFI

uint32_t ffi_init();

typedef struct _ffi_type {
size_t size;
unsigned short alignment;
unsigned short type;
struct _ffi_type **elements;
} ffi_type;

typedef enum {
FFI_OK = 0,
FFI_BAD_TYPEDEF,
FFI_BAD_ABI,
FFI_BAD_ARGTYPE
} ffi_status;

// These are target depenent so we set them manually for each ABI by referencing
// the FFI source.
typedef enum ffi_abi {
#if (defined(_M_X64) || defined(__x86_64__))
FFI_DEFAULT_ABI = 2, // FFI_UNIX64.
#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64)
FFI_DEFAULT_ABI = 1, // FFI_SYSV.
#elif defined(__powerpc64__)
FFI_DEFAULT_ABI = 8, // FFI_LINUX.
#elif defined(__s390x__)
FFI_DEFAULT_ABI = 1, // FFI_SYSV.
#else
#error "Unknown ABI"
#endif
} ffi_cif;

#ifdef __cplusplus
extern "C" {
#endif

#define FFI_EXTERN extern
#define FFI_API

FFI_EXTERN ffi_type ffi_type_void;
FFI_EXTERN ffi_type ffi_type_pointer;

FFI_API
void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);

FFI_API
ffi_status ffi_prep_cif(ffi_cif *cif, ffi_abi abi, unsigned int nargs,
ffi_type *rtype, ffi_type **atypes);

#ifdef __cplusplus
}
#endif

#endif // DYNAMIC_FFI_FFI_H
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,11 @@ struct GenELF64PluginTy final : public GenericPluginTy {
ompt::connectLibrary();
#endif

#ifdef USES_DYNAMIC_FFI
if (auto Err = Plugin::check(ffi_init(), "Failed to initialize libffi"))
return std::move(Err);
#endif

return NUM_DEVICES;
}

Expand Down

0 comments on commit c7c68f1

Please sign in to comment.