Skip to content

Commit 5b67bce

Browse files
committed
[OpenMP] [OMPT] [2/8] Implemented a connector for communication of OMPT callbacks between libraries.
This is part of a set of patches implementing OMPT target callback support and has been split out of the originally submitted https://reviews.llvm.org/D113728. The overall design can be found in https://rice.app.box.com/s/pf3gix2hs4d4o1aatwir1set05xmjljc The purpose of this patch is to provide a way to register tool-provided callbacks into libomp when libomptarget is loaded. Introduced a cmake variable LIBOMPTARGET_OMPT_SUPPORT that can be used to control OMPT target support. It follows host OMPT support, controlled by LIBOMP_HAVE_OMPT_SUPPORT. Added a connector that can be used to communicate between OMPT implementations in libomp and libomptarget or libomptarget and a plugin. Added a global constructor in libomptarget that uses the connector to force registration of tool-provided callbacks in libomp. A pair of init and fini functions are provided to libomp as part of the connect process which will be used to register the tool-provided callbacks in libomptarget. Patch from John Mellor-Crummey <johnmc@rice.edu> (With contributions from Dhruva Chakrabarti <Dhruva.Chakrabarti@amd.com>) Reviewed By: dreachem, jhuber6 Differential Revision: https://reviews.llvm.org/D123572
1 parent 628fdc3 commit 5b67bce

File tree

11 files changed

+272
-7
lines changed

11 files changed

+272
-7
lines changed

openmp/libomptarget/CMakeLists.txt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,28 @@ if(LIBOMPTARGET_ENABLE_DEBUG)
6969
add_definitions(-DOMPTARGET_DEBUG)
7070
endif()
7171

72+
# OMPT support for libomptarget
73+
# Follow host OMPT support and check if host support has been requested.
74+
# LIBOMP_HAVE_OMPT_SUPPORT indicates whether host OMPT support has been implemented.
75+
# LIBOMP_OMPT_SUPPORT indicates whether host OMPT support has been requested (default is ON).
76+
# LIBOMPTARGET_OMPT_SUPPORT indicates whether target OMPT support has been requested (default is ON).
77+
set(OMPT_TARGET_DEFAULT FALSE)
78+
if ((LIBOMP_HAVE_OMPT_SUPPORT) AND (LIBOMP_OMPT_SUPPORT) AND (NOT WIN32))
79+
set (OMPT_TARGET_DEFAULT TRUE)
80+
endif()
81+
set(LIBOMPTARGET_OMPT_SUPPORT ${OMPT_TARGET_DEFAULT} CACHE BOOL "OMPT-target-support?")
82+
if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT))
83+
add_definitions(-DOMPT_SUPPORT=1)
84+
message(STATUS "OMPT target enabled")
85+
else()
86+
message(STATUS "OMPT target disabled")
87+
endif()
88+
89+
pythonize_bool(LIBOMPTARGET_OMPT_SUPPORT)
90+
7291
set(LIBOMPTARGET_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
92+
message(STATUS "OpenMP tools dir in libomptarget: ${LIBOMP_OMP_TOOLS_INCLUDE_DIR}")
93+
include_directories(${LIBOMP_OMP_TOOLS_INCLUDE_DIR})
7394

7495
# Build target agnostic offloading library.
7596
set(LIBOMPTARGET_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
//=== ompt_connector.h - Target independent OpenMP target RTL -- C++ ------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// Support used by OMPT implementation to establish communication between
10+
// various OpenMP runtime libraries: host openmp library, target-independent
11+
// runtime library, and device-dependent runtime libraries.
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
#ifndef _OMPT_CONNECTOR_H
16+
#define _OMPT_CONNECTOR_H
17+
18+
#ifdef OMPT_SUPPORT
19+
20+
#include "llvm/Support/DynamicLibrary.h"
21+
22+
#include <memory>
23+
#include <string>
24+
25+
#include "Debug.h"
26+
#include "omp-tools.h"
27+
#include "omptarget.h"
28+
29+
#define LIBOMPTARGET_STRINGIFY(s) #s
30+
31+
/// Type for the function to be invoked for connecting two libraries.
32+
typedef void (*OmptConnectRtnTy)(ompt_start_tool_result_t *result);
33+
34+
/// Establish connection between openmp runtime libraries
35+
///
36+
/// This class is used to communicate between an OMPT implementation in
37+
/// libomptarget and libomp. It is also used to communicate between an
38+
/// OMPT implementation in a device-specific plugin and
39+
/// libomptarget. The decision whether OMPT is enabled or not needs to
40+
/// be made when the library is loaded before any functions in the
41+
/// library are invoked. For that reason, an instance of this class is
42+
/// intended to be defined in the constructor for libomptarget or a
43+
/// plugin so that the decision about whether OMPT is supposed to be
44+
/// enabled is known before any interface function in the library is
45+
/// invoked.
46+
class OmptLibraryConnectorTy {
47+
public:
48+
/// Use \p LibName as the prefix of the global function used for connecting
49+
/// two libraries, the source indicated by \p LibName and the destination
50+
/// being the one that creates this object.
51+
OmptLibraryConnectorTy(const char *Ident) {
52+
LibIdent.append(Ident);
53+
IsInitialized = false;
54+
}
55+
OmptLibraryConnectorTy() = delete;
56+
/// Use \p OmptResult init to connect the two libraries denoted by this
57+
/// object. The init function of \p OmptResult will be used during connection
58+
/// and the fini function of \p OmptResult will be used during teardown.
59+
void connect(ompt_start_tool_result_t *OmptResult) {
60+
initialize();
61+
if (!LibConnHandle)
62+
return;
63+
// Call the function provided by the source library for connect
64+
LibConnHandle(OmptResult);
65+
}
66+
67+
private:
68+
void initialize() {
69+
if (IsInitialized)
70+
return;
71+
72+
std::string ErrMsg;
73+
std::string LibName = LibIdent;
74+
LibName += ".so";
75+
76+
DP("OMPT: Trying to load library %s\n", LibName.c_str());
77+
auto DynLibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
78+
llvm::sys::DynamicLibrary::getPermanentLibrary(LibName.c_str(),
79+
&ErrMsg));
80+
if (!DynLibHandle->isValid()) {
81+
// The upper layer will bail out if the handle is null.
82+
LibConnHandle = nullptr;
83+
} else {
84+
auto LibConnRtn = "ompt_" + LibIdent + "_connect";
85+
DP("OMPT: Trying to get address of connection routine %s\n",
86+
LibConnRtn.c_str());
87+
LibConnHandle = reinterpret_cast<OmptConnectRtnTy>(
88+
DynLibHandle->getAddressOfSymbol(LibConnRtn.c_str()));
89+
}
90+
DP("OMPT: Library connection handle = %p\n", LibConnHandle);
91+
IsInitialized = true;
92+
}
93+
94+
/// Ensure initialization occurs only once
95+
bool IsInitialized;
96+
/// Handle of connect routine provided by source library
97+
OmptConnectRtnTy LibConnHandle;
98+
/// Name of connect routine provided by source library
99+
std::string LibIdent;
100+
};
101+
102+
#endif // OMPT_SUPPORT
103+
104+
#endif // _OMPT_CONNECTOR_H

openmp/libomptarget/src/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ add_llvm_library(omptarget
2020
interface.cpp
2121
interop.cpp
2222
omptarget.cpp
23+
ompt_callback.cpp
2324
rtl.cpp
2425
LegacyAPI.cpp
2526

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
//===-- ompt_callback.cpp - Target independent OpenMP target RTL -- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// Implementation of OMPT callback interfaces for target independent layer
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifdef OMPT_SUPPORT
14+
15+
#include <assert.h>
16+
#include <atomic>
17+
#include <cstdlib>
18+
#include <cstring>
19+
20+
#include "omp-tools.h"
21+
#include "ompt_connector.h"
22+
#include "private.h"
23+
24+
#define fnptr_to_ptr(x) ((void *)(uint64_t)x)
25+
26+
/// Used to indicate whether OMPT was enabled for this library
27+
bool ompt_enabled = false;
28+
29+
/// This is the function called by the higher layer (libomp) responsible
30+
/// for initializing OMPT in this library. This is passed to libomp
31+
/// as part of the OMPT connector object.
32+
/// \p lookup to be used to query callbacks registered with libomp
33+
/// \p initial_device_num Initial device num provided by libomp
34+
/// \p tool_data as provided by the tool
35+
static int ompt_libomptarget_initialize(ompt_function_lookup_t lookup,
36+
int initial_device_num,
37+
ompt_data_t *tool_data) {
38+
DP("enter ompt_libomptarget_initialize!\n");
39+
ompt_enabled = true;
40+
// TODO use the parameters to populate callbacks in libomptarget
41+
DP("exit ompt_libomptarget_initialize!\n");
42+
return 0;
43+
}
44+
45+
static void ompt_libomptarget_finalize(ompt_data_t *data) {
46+
DP("enter ompt_libomptarget_finalize!\n");
47+
ompt_enabled = false;
48+
DP("exit ompt_libomptarget_finalize!\n");
49+
}
50+
51+
/*****************************************************************************
52+
* constructor
53+
*****************************************************************************/
54+
/// Used to initialize callbacks implemented by the tool. This interface
55+
/// will lookup the callbacks table in libomp and assign them to the callbacks
56+
/// maintained in libomptarget. Using priority 102 to have this constructor
57+
/// run after the init target library constructor with priority 101 (see
58+
/// rtl.cpp).
59+
__attribute__((constructor(102))) static void ompt_init(void) {
60+
DP("OMPT: Enter ompt_init\n");
61+
// Connect with libomp
62+
static OmptLibraryConnectorTy LibompConnector("libomp");
63+
static ompt_start_tool_result_t OmptResult;
64+
65+
// Initialize OmptResult with the init and fini functions that will be
66+
// called by the connector
67+
OmptResult.initialize = ompt_libomptarget_initialize;
68+
OmptResult.finalize = ompt_libomptarget_finalize;
69+
OmptResult.tool_data.value = 0;
70+
71+
// Now call connect that causes the above init/fini functions to be called
72+
LibompConnector.connect(&OmptResult);
73+
DP("OMPT: Exit ompt_init\n");
74+
}
75+
76+
#endif // OMPT_SUPPORT

openmp/runtime/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,7 @@ add_subdirectory(test)
419419
# make these variables available for tools:
420420
set(LIBOMP_LIBRARY_DIR ${LIBOMP_LIBRARY_DIR} PARENT_SCOPE)
421421
set(LIBOMP_INCLUDE_DIR ${LIBOMP_INCLUDE_DIR} PARENT_SCOPE)
422+
set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_OMP_TOOLS_INCLUDE_DIR} PARENT_SCOPE)
422423
# make these variables available for tools/libompd:
423424
set(LIBOMP_SRC_DIR ${LIBOMP_SRC_DIR} PARENT_SCOPE)
424425
set(LIBOMP_OMPD_SUPPORT ${LIBOMP_OMPD_SUPPORT} PARENT_SCOPE)

openmp/runtime/cmake/config-ix.cmake

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,8 @@ else()
331331
endif()
332332
endif()
333333

334+
set(LIBOMP_HAVE_OMPT_SUPPORT ${LIBOMP_HAVE_OMPT_SUPPORT} PARENT_SCOPE)
335+
334336
# Check if HWLOC support is available
335337
if(${LIBOMP_USE_HWLOC})
336338
find_path(LIBOMP_HWLOC_INCLUDE_DIR NAMES hwloc.h HINTS ${LIBOMP_HWLOC_INSTALL_DIR} PATH_SUFFIXES include)

openmp/runtime/src/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,7 @@ if(${LIBOMP_OMPT_SUPPORT})
391391
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH})
392392
# install under legacy name ompt.h
393393
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
394+
set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
394395
endif()
395396
if(${LIBOMP_FORTRAN_MODULES})
396397
install(FILES

openmp/runtime/src/exports_so.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ VERSION {
2626
# OMPT API
2727
#
2828
ompt_start_tool; # OMPT start interface
29+
ompt_libomp_connect; # OMPT libomptarget interface
2930

3031
ompc_*; # omp.h renames some standard functions to ompc_*.
3132
kmp_*; # Intel extensions.

openmp/runtime/src/ompt-general.cpp

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,9 @@ static void *ompt_tool_module = NULL;
110110
#define OMPT_DLCLOSE(Lib) dlclose(Lib)
111111
#endif
112112

113+
/// Used to track the initializer and the finalizer provided by libomptarget
114+
static ompt_start_tool_result_t *libomptarget_ompt_result = NULL;
115+
113116
/*****************************************************************************
114117
* forward declarations
115118
****************************************************************************/
@@ -456,7 +459,7 @@ void ompt_pre_init() {
456459
if (verbose_init && verbose_file != stderr && verbose_file != stdout)
457460
fclose(verbose_file);
458461
#if OMPT_DEBUG
459-
printf("ompt_pre_init(): ompt_enabled = %d\n", ompt_enabled);
462+
printf("ompt_pre_init(): ompt_enabled = %d\n", ompt_enabled.enabled);
460463
#endif
461464
}
462465

@@ -509,12 +512,13 @@ void ompt_post_init() {
509512
}
510513

511514
void ompt_fini() {
512-
if (ompt_enabled.enabled
513-
#if OMPD_SUPPORT
514-
&& ompt_start_tool_result && ompt_start_tool_result->finalize
515-
#endif
516-
) {
517-
ompt_start_tool_result->finalize(&(ompt_start_tool_result->tool_data));
515+
if (ompt_enabled.enabled) {
516+
if (ompt_start_tool_result && ompt_start_tool_result->finalize) {
517+
ompt_start_tool_result->finalize(&(ompt_start_tool_result->tool_data));
518+
}
519+
if (libomptarget_ompt_result && libomptarget_ompt_result->finalize) {
520+
libomptarget_ompt_result->finalize(NULL);
521+
}
518522
}
519523

520524
if (ompt_tool_module)
@@ -869,5 +873,50 @@ static ompt_interface_fn_t ompt_fn_lookup(const char *s) {
869873

870874
FOREACH_OMPT_INQUIRY_FN(ompt_interface_fn)
871875

876+
#undef ompt_interface_fn
877+
872878
return NULL;
873879
}
880+
881+
/// Lookup function to query libomp callbacks registered by the tool
882+
static ompt_interface_fn_t ompt_libomp_target_fn_lookup(const char *s) {
883+
#define ompt_interface_fn(fn, type, code) \
884+
if (strcmp(s, #fn) == 0) \
885+
return (ompt_interface_fn_t)ompt_callbacks.ompt_callback(fn);
886+
887+
FOREACH_OMPT_DEVICE_EVENT(ompt_interface_fn)
888+
FOREACH_OMPT_EMI_EVENT(ompt_interface_fn)
889+
FOREACH_OMPT_NOEMI_EVENT(ompt_interface_fn)
890+
891+
#undef ompt_interface_fn
892+
893+
return (ompt_interface_fn_t)0;
894+
}
895+
896+
/// This function is called by the libomptarget connector to assign
897+
/// callbacks already registered with libomp.
898+
_OMP_EXTERN void ompt_libomp_connect(ompt_start_tool_result_t *result) {
899+
OMPT_VERBOSE_INIT_PRINT("libomp --> OMPT: Enter libomp_ompt_connect\n");
900+
901+
// Ensure libomp callbacks have been added if not already
902+
__ompt_force_initialization();
903+
904+
if (ompt_enabled.enabled &&
905+
// Callbacks are initiated only if the device initialize callback
906+
// has been registered by the tool
907+
ompt_callbacks.ompt_callback(ompt_callback_device_initialize)) {
908+
if (result) {
909+
OMPT_VERBOSE_INIT_PRINT(
910+
"libomp --> OMPT: Connecting with libomptarget\n");
911+
// Pass in the libomp lookup function so that the already registered
912+
// functions can be extracted and assigned to the callbacks in
913+
// libomptarget
914+
result->initialize(ompt_libomp_target_fn_lookup,
915+
0 /* initial_device_num */, nullptr /* tool_data */);
916+
// Track the object provided by libomptarget so that the finalizer can be
917+
// called during OMPT finalization
918+
libomptarget_ompt_result = result;
919+
}
920+
}
921+
OMPT_VERBOSE_INIT_PRINT("libomp --> OMPT: Exit libomp_ompt_connect\n");
922+
}

openmp/runtime/src/ompt-specific.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,11 @@ ompt_task_info_t *__ompt_get_scheduling_taskinfo(int depth) {
188188
//******************************************************************************
189189
// interface operations
190190
//******************************************************************************
191+
//----------------------------------------------------------
192+
// initialization support
193+
//----------------------------------------------------------
194+
195+
void __ompt_force_initialization() { __kmp_serial_initialize(); }
191196

192197
//----------------------------------------------------------
193198
// thread support

0 commit comments

Comments
 (0)