Skip to content

Commit

Permalink
[OMPT] Provide initialization for Mac OS X
Browse files Browse the repository at this point in the history 
Traditionally, the library had a weak symbol for ompt_start_tool()
that served as fallback and disabled OMPT if called. Tools could
provide their own version and replace the default implementation
to register callbacks and lookup functions. This mechanism has
worked reasonably well on Linux systems where this interface was
initially developed.

On Darwin / Mac OS X the situation is a bit more complicated and
the weak symbol doesn't work out-of-the-box. In my tests, the
library with the tool needed to link against the OpenMP runtime
to make the process work. This would effectively mean that a tool
needed to choose a runtime library whereas one design goal of the
interface was to allow tools that are agnostic of the runtime.

The solution is to use dlsym() with the argument RTLD_DEFAULT so
that static implementations of ompt_start_tool() are found in the
main executable. This works because the linker on Mac OS X includes
all symbols of an executable in the global symbol table by default.
To use the same code path on Linux, the application would need to
be built with -Wl,--export-dynamic. To avoid this restriction, we
continue to use weak symbols on Linux systems as before.

Finally this patch extends the existing test to cover all possible
ways of initializing the tool as described by the standard. It
also fixes ompt_finalize() to not call omp_get_thread_num() when
the library is shut down which resulted in hangs on Darwin.
The changes have been tested on Linux to make sure that it passes
the current tests as well as the newly extended one.

Differential Revision: https://reviews.llvm.org/D39801

llvm-svn: 317980
  • Loading branch information
@hahnjo
hahnjo committed Nov 11, 2017
1 parent 3acaa70 commit d0ef19e
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 49 deletions.
4 changes: 2 additions & 2 deletions openmp/runtime/cmake/config-ix.cmake
View file
Expand Up @@ -223,7 +223,7 @@ endif()

# Check if OMPT support is available
# Currently, __builtin_frame_address() is required for OMPT
# Weak attribute is required for Unices, LIBPSAPI is used for Windows
# Weak attribute is required for Unices (except Darwin), LIBPSAPI is used for Windows
check_c_source_compiles("int main(int argc, char** argv) {
void* p = __builtin_frame_address(0);
return 0;}" LIBOMP_HAVE___BUILTIN_FRAME_ADDRESS)
Expand All @@ -238,7 +238,7 @@ endif()
if(NOT LIBOMP_HAVE___BUILTIN_FRAME_ADDRESS)
set(LIBOMP_HAVE_OMPT_SUPPORT FALSE)
else()
if(LIBOMP_HAVE_WEAK_ATTRIBUTE OR LIBOMP_HAVE_PSAPI)
if((WIN32 AND LIBOMP_HAVE_PSAPI) OR APPLE OR (NOT WIN32 AND LIBOMP_HAVE_WEAK_ATTRIBUTE))
set(LIBOMP_HAVE_OMPT_SUPPORT TRUE)
else()
set(LIBOMP_HAVE_OMPT_SUPPORT FALSE)
Expand Down
86 changes: 53 additions & 33 deletions openmp/runtime/src/ompt-general.cpp
View file
Expand Up @@ -88,51 +88,62 @@ OMPT_API_ROUTINE ompt_data_t *ompt_get_thread_data(void);
* initialization and finalization (private operations)
****************************************************************************/

/* On Unix-like systems that support weak symbols the following implementation
* of ompt_start_tool() will be used in case no tool-supplied implementation of
* this function is present in the address space of a process.
*
* On Windows, the ompt_tool_windows function is used to find the
* ompt_tool symbol across all modules loaded by a process. If ompt_tool is
* found, ompt_tool's return value is used to initialize the tool. Otherwise,
* NULL is returned and OMPT won't be enabled */

typedef ompt_start_tool_result_t *(*ompt_start_tool_t)(unsigned int,
const char *);

#if KMP_OS_UNIX
#if KMP_OS_DARWIN

#if OMPT_HAVE_WEAK_ATTRIBUTE
_OMP_EXTERN OMPT_WEAK_ATTRIBUTE
#elif defined KMP_DYNAMIC_LIB
_OMP_EXTERN
#warning Activation of OMPT is might fail for tools statically linked into the application.
#else
#error Activation of OMPT is not supported on this platform.
#endif
ompt_start_tool_result_t *
// While Darwin supports weak symbols, the library that wishes to provide a new
// implementation has to link against this runtime which defeats the purpose
// of having tools that are agnostic of the underlying runtime implementation.
//
// Fortunately, the linker includes all symbols of an executable in the global
// symbol table by default so dlsym() even finds static implementations of
// ompt_start_tool. For this to work on Linux, -Wl,--export-dynamic needs to be
// passed when building the application which we don't want to rely on.

static ompt_start_tool_result_t *ompt_tool_darwin(unsigned int omp_version,
const char *runtime_version) {
ompt_start_tool_result_t *ret = NULL;
// Search symbol in the current address space.
ompt_start_tool_t start_tool =
(ompt_start_tool_t)dlsym(RTLD_DEFAULT, "ompt_start_tool");
if (start_tool) {
ret = start_tool(omp_version, runtime_version);
}
return ret;
}

#elif OMPT_HAVE_WEAK_ATTRIBUTE

// On Unix-like systems that support weak symbols the following implementation
// of ompt_start_tool() will be used in case no tool-supplied implementation of
// this function is present in the address space of a process.

_OMP_EXTERN OMPT_WEAK_ATTRIBUTE ompt_start_tool_result_t *
ompt_start_tool(unsigned int omp_version, const char *runtime_version) {
#ifdef KMP_DYNAMIC_LIB
ompt_start_tool_result_t *ret = NULL;
// Try next symbol in the address space
// Search next symbol in the current address space. This can happen if the
// runtime library is linked before the tool. Since glibc 2.2 strong symbols
// don't override weak symbols that have been found before unless the user
// sets the environment variable LD_DYNAMIC_WEAK.
ompt_start_tool_t next_tool =
(ompt_start_tool_t)dlsym(RTLD_NEXT, "ompt_start_tool");
if (next_tool)
ret = (next_tool)(omp_version, runtime_version);
if (next_tool) {
ret = next_tool(omp_version, runtime_version);
}
return ret;
#else
#if OMPT_DEBUG
printf("ompt_start_tool() is called from the RTL\n");
#endif
return NULL;
#endif
}

#elif OMPT_HAVE_PSAPI

// On Windows, the ompt_tool_windows function is used to find the
// ompt_start_tool symbol across all modules loaded by a process. If
// ompt_start_tool is found, ompt_start_tool's return value is used to
// initialize the tool. Otherwise, NULL is returned and OMPT won't be enabled.

#include <psapi.h>
#pragma comment(lib, "psapi.lib")
#define ompt_start_tool ompt_tool_windows

// The number of loaded modules to start enumeration with EnumProcessModules()
#define NUM_MODULES 128
Expand Down Expand Up @@ -193,8 +204,8 @@ ompt_tool_windows(unsigned int omp_version, const char *runtime_version) {
return NULL;
}
#else
#error Either __attribute__((weak)) or psapi.dll are required for OMPT support
#endif // OMPT_HAVE_WEAK_ATTRIBUTE
#error Activation of OMPT is not supported on this platform.
#endif

static ompt_start_tool_result_t *
ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
Expand All @@ -208,7 +219,16 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
#endif

// Try in the current address space
if ((ret = ompt_start_tool(omp_version, runtime_version)))
#if KMP_OS_DARWIN
ret = ompt_tool_darwin(omp_version, runtime_version);
#elif OMPT_HAVE_WEAK_ATTRIBUTE
ret = ompt_start_tool(omp_version, runtime_version);
#elif OMPT_HAVE_PSAPI
ret = ompt_tool_windows(omp_version, runtime_version);
#else
#error Activation of OMPT is not supported on this platform.
#endif
if (ret)
return ret;

// Try tool-libraries-var ICV
Expand Down
16 changes: 10 additions & 6 deletions openmp/runtime/test/lit.cfg
View file
Expand Up @@ -42,8 +42,7 @@ config.test_exec_root = config.libomp_obj_root
config.test_format = lit.formats.ShTest()

# compiler flags
config.test_cflags = config.test_openmp_flag + \
" -I " + config.test_source_root + \
config.test_cflags = " -I " + config.test_source_root + \
" -I " + config.omp_header_directory + \
" -L " + config.library_dir + \
" " + config.test_extra_cflags
Expand Down Expand Up @@ -106,11 +105,9 @@ config.substitutions.append(("%libomp-compile-and-run", \
config.substitutions.append(("%libomp-cxx-compile-and-run", \
"%libomp-cxx-compile && %libomp-run"))
config.substitutions.append(("%libomp-cxx-compile", \
"%clangXX %cflags -std=c++11 %s -o %t" + libs))
"%clangXX %openmp_flag %cflags -std=c++11 %s -o %t" + libs))
config.substitutions.append(("%libomp-compile", \
"%clang %cflags %s -o %t" + libs))
config.substitutions.append(("%libomp-tool", \
"%clang %cflags -shared -fPIC -o %T/tool.so" + libs))
"%clang %openmp_flag %cflags %s -o %t" + libs))
config.substitutions.append(("%libomp-run", "%t"))
config.substitutions.append(("%clangXX", config.test_cxx_compiler))
config.substitutions.append(("%clang", config.test_compiler))
Expand All @@ -120,3 +117,10 @@ config.substitutions.append(("%cflags", config.test_cflags))
if config.has_ompt:
config.substitutions.append(("FileCheck", config.test_filecheck))
config.substitutions.append(("%sort-threads", "sort --numeric-sort --stable"))
if config.operating_system == 'Windows':
# No such environment variable on Windows.
config.substitutions.append(("%preload-tool", "true ||"))
elif config.operating_system == 'Darwin':
config.substitutions.append(("%preload-tool", "env DYLD_INSERT_LIBRARIES=%T/tool.so"))
else:
config.substitutions.append(("%preload-tool", "env LD_PRELOAD=%T/tool.so"))
31 changes: 23 additions & 8 deletions openmp/runtime/test/ompt/loadtool/tool_available.c
View file
@@ -1,13 +1,28 @@
// RUN: %libomp-compile -DCODE && %libomp-compile -DTOOL -o%T/tool.so -shared -fPIC && env OMP_TOOL_LIBRARIES=%T/tool.so %libomp-run | FileCheck %s
// The OpenMP standard defines 3 ways of providing ompt_start_tool:
// 1. "statically-linking the tool’s definition of ompt_start_tool into an OpenMP application"
// RUN: %libomp-compile -DCODE -DTOOL && %libomp-run | FileCheck %s

// Note: We should compile the tool without -fopenmp as other tools developer
// would do. Otherwise this test may pass for the wrong reasons on Darwin.
// RUN: %clang %cflags -DTOOL -shared -fPIC %s -o %T/tool.so
// 2. "introducing a dynamically-linked library that includes the tool’s definition of ompt_start_tool into the application’s address space"
// 2.1 Link with tool during compilation
// RUN: %libomp-compile -DCODE %T/tool.so && %libomp-run | FileCheck %s
// 2.2 Link with tool during compilation, but AFTER the runtime
// RUN: %libomp-compile -DCODE -lomp %T/tool.so && %libomp-run | FileCheck %s
// 2.3 Inject tool via the dynamic loader
// RUN: %libomp-compile -DCODE && %preload-tool %libomp-run | FileCheck %s

// 3. "providing the name of a dynamically-linked library appropriate for the architecture and operating system used by the application in the tool-libraries-var ICV"
// RUN: %libomp-compile -DCODE && env OMP_TOOL_LIBRARIES=%T/tool.so %libomp-run | FileCheck %s

// REQUIRES: ompt

/*
* This file contains code for an OMPT shared library tool to be
* loaded and the code for the OpenMP executable.
* This file contains code for an OMPT shared library tool to be
* loaded and the code for the OpenMP executable.
* -DTOOL enables the code for the tool during compilation
* -DCODE enables the code for the executable during compilation
* The RUN line compiles the two binaries and then tries to load
* the tool using the OMP_TOOL_LIBRARIES environmental variable.
*/

#ifdef CODE
Expand All @@ -21,8 +36,8 @@ int main()


// Check if libomp supports the callbacks for this test.
// CHECK-NOT: {{^}}0: Could not register callback
// CHECK-NOT: {{^}}0: Could not register callback

// CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
// CHECK: {{^}}0: ompt_event_runtime_shutdown

Expand All @@ -46,7 +61,7 @@ int ompt_initialize(

void ompt_finalize(ompt_data_t* tool_data)
{
printf("%d: ompt_event_runtime_shutdown\n", omp_get_thread_num());
printf("0: ompt_event_runtime_shutdown\n");
}

ompt_start_tool_result_t* ompt_start_tool(
Expand Down

0 comments on commit d0ef19e

Please sign in to comment.