44 changes: 27 additions & 17 deletions compiler-rt/lib/lsan/lsan_interceptors.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

#include "interception/interception.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_report.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
Expand Down Expand Up @@ -43,6 +44,22 @@ int pthread_key_create(unsigned *key, void (*destructor)(void* v));
int pthread_setspecific(unsigned key, const void *v);
}

struct DlsymAlloc : DlSymAllocator<DlsymAlloc> {
static bool UseImpl() { return lsan_init_is_running; }
static void OnAllocate(const void *ptr, uptr size) {
#if CAN_SANITIZE_LEAKS
// Suppress leaks from dlerror(). Previously dlsym hack on global array was
// used by leak sanitizer as a root region.
__lsan_register_root_region(ptr, size);
#endif
}
static void OnFree(const void *ptr, uptr size) {
#if CAN_SANITIZE_LEAKS
__lsan_unregister_root_region(ptr, size);
#endif
}
};

///// Malloc/free interceptors. /////

namespace std {
Expand All @@ -52,41 +69,34 @@ namespace std {

#if !SANITIZER_MAC
INTERCEPTOR(void*, malloc, uptr size) {
if (DlsymAlloc::Use())
return DlsymAlloc::Allocate(size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_malloc(size, stack);
}

INTERCEPTOR(void, free, void *p) {
if (DlsymAlloc::PointerIsMine(p))
return DlsymAlloc::Free(p);
ENSURE_LSAN_INITED;
lsan_free(p);
}

INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
// This hack is not required for Fuchsia because there are no dlsym calls
// involved in setting up interceptors.
#if !SANITIZER_FUCHSIA
if (lsan_init_is_running) {
// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
const uptr kCallocPoolSize = 1024;
static uptr calloc_memory_for_dlsym[kCallocPoolSize];
static uptr allocated;
uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
void *mem = (void*)&calloc_memory_for_dlsym[allocated];
allocated += size_in_words;
CHECK(allocated < kCallocPoolSize);
return mem;
}
#endif // !SANITIZER_FUCHSIA
if (DlsymAlloc::Use())
return DlsymAlloc::Callocate(nmemb, size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_calloc(nmemb, size, stack);
}

INTERCEPTOR(void*, realloc, void *q, uptr size) {
INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Realloc(ptr, size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_realloc(q, size, stack);
return lsan_realloc(ptr, size, stack);
}

INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) {
Expand Down
80 changes: 14 additions & 66 deletions compiler-rt/lib/memprof/memprof_malloc_linux.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,104 +23,52 @@
#include "memprof_internal.h"
#include "memprof_stack.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"

// ---------------------- Replacement functions ---------------- {{{1
using namespace __memprof;

static uptr allocated_for_dlsym;
static uptr last_dlsym_alloc_size_in_words;
static const uptr kDlsymAllocPoolSize = 1024;
static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];

static inline bool IsInDlsymAllocPool(const void *ptr) {
uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
}

static void *AllocateFromLocalPool(uptr size_in_bytes) {
uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
last_dlsym_alloc_size_in_words = size_in_words;
allocated_for_dlsym += size_in_words;
CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
return mem;
}

static void DeallocateFromLocalPool(const void *ptr) {
// Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
// error messages and instead uses malloc followed by free. To avoid pool
// exhaustion due to long object filenames, handle that special case here.
uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset];
if (prev_mem == ptr) {
REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
allocated_for_dlsym = prev_offset;
last_dlsym_alloc_size_in_words = 0;
}
}

static inline bool MaybeInDlsym() { return memprof_init_is_running; }

static inline bool UseLocalPool() { return MaybeInDlsym(); }

static void *ReallocFromLocalPool(void *ptr, uptr size) {
const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
void *new_ptr;
if (UNLIKELY(UseLocalPool())) {
new_ptr = AllocateFromLocalPool(size);
} else {
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
new_ptr = memprof_malloc(size, &stack);
}
internal_memcpy(new_ptr, ptr, copy_size);
return new_ptr;
}
struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
static bool UseImpl() { return memprof_init_is_running; }
};

INTERCEPTOR(void, free, void *ptr) {
if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
DeallocateFromLocalPool(ptr);
return;
}
if (DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
memprof_free(ptr, &stack, FROM_MALLOC);
}

#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void, cfree, void *ptr) {
if (UNLIKELY(IsInDlsymAllocPool(ptr)))
return;
if (DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
memprof_free(ptr, &stack, FROM_MALLOC);
}
#endif // SANITIZER_INTERCEPT_CFREE

INTERCEPTOR(void *, malloc, uptr size) {
if (UNLIKELY(UseLocalPool()))
// Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
return AllocateFromLocalPool(size);
if (DlsymAlloc::Use())
return DlsymAlloc::Allocate(size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_malloc(size, &stack);
}

INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) {
if (UNLIKELY(UseLocalPool()))
// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
return AllocateFromLocalPool(nmemb * size);
if (DlsymAlloc::Use())
return DlsymAlloc::Callocate(nmemb, size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_calloc(nmemb, size, &stack);
}

INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
if (UNLIKELY(IsInDlsymAllocPool(ptr)))
return ReallocFromLocalPool(ptr, size);
if (UNLIKELY(UseLocalPool()))
return AllocateFromLocalPool(size);
if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Realloc(ptr, size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_realloc(ptr, size, &stack);
Expand Down
57 changes: 18 additions & 39 deletions compiler-rt/lib/msan/msan_interceptors.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
#include "msan_report.h"
#include "msan_thread.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_errno.h"
Expand Down Expand Up @@ -75,22 +75,9 @@ bool IsInInterceptorScope() {
return in_interceptor_scope;
}

static uptr allocated_for_dlsym;
static const uptr kDlsymAllocPoolSize = 1024;
static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];

static bool IsInDlsymAllocPool(const void *ptr) {
uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
return off < sizeof(alloc_memory_for_dlsym);
}

static void *AllocateFromLocalPool(uptr size_in_bytes) {
uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
allocated_for_dlsym += size_in_words;
CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
return mem;
}
struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
static bool UseImpl() { return !msan_inited; }
};

#define ENSURE_MSAN_INITED() do { \
CHECK(!msan_init_is_running); \
Expand Down Expand Up @@ -221,14 +208,20 @@ INTERCEPTOR(void *, pvalloc, SIZE_T size) {
#endif

INTERCEPTOR(void, free, void *ptr) {
if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
if (UNLIKELY(!ptr))
return;
if (DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
MsanDeallocate(&stack, ptr);
}

#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void, cfree, void *ptr) {
if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
if (UNLIKELY(!ptr))
return;
if (DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
MsanDeallocate(&stack, ptr);
}
Expand Down Expand Up @@ -879,27 +872,14 @@ INTERCEPTOR(int, epoll_pwait, int epfd, void *events, int maxevents,

INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
if (UNLIKELY(!msan_inited))
// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
return AllocateFromLocalPool(nmemb * size);
if (DlsymAlloc::Use())
return DlsymAlloc::Callocate(nmemb, size);
return msan_calloc(nmemb, size, &stack);
}

INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
void *new_ptr;
if (UNLIKELY(!msan_inited)) {
new_ptr = AllocateFromLocalPool(copy_size);
} else {
copy_size = size;
GET_MALLOC_STACK_TRACE;
new_ptr = msan_malloc(copy_size, &stack);
}
internal_memcpy(new_ptr, ptr, copy_size);
return new_ptr;
}
if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Realloc(ptr, size);
GET_MALLOC_STACK_TRACE;
return msan_realloc(ptr, size, &stack);
}
Expand All @@ -910,9 +890,8 @@ INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) {
}

INTERCEPTOR(void *, malloc, SIZE_T size) {
if (UNLIKELY(!msan_inited))
// Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
return AllocateFromLocalPool(size);
if (DlsymAlloc::Use())
return DlsymAlloc::Allocate(size);
GET_MALLOC_STACK_TRACE;
return msan_malloc(size, &stack);
}
Expand Down
1 change: 1 addition & 0 deletions compiler-rt/lib/sanitizer_common/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ set(SANITIZER_IMPL_HEADERS
sanitizer_allocator.h
sanitizer_allocator_checks.h
sanitizer_allocator_combined.h
sanitizer_allocator_dlsym.h
sanitizer_allocator_interface.h
sanitizer_allocator_internal.h
sanitizer_allocator_local_cache.h
Expand Down
79 changes: 79 additions & 0 deletions compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
//===-- sanitizer_allocator_dlsym.h -----------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Hack: Sanitizer initializer calls dlsym which may need to allocate and call
// back into uninitialized sanitizer.
//
//===----------------------------------------------------------------------===//

#ifndef SANITIZER_ALLOCATOR_DLSYM_H
#define SANITIZER_ALLOCATOR_DLSYM_H

#include "sanitizer_allocator_internal.h"

namespace __sanitizer {

template <typename Details>
struct DlSymAllocator {
static bool Use() {
// Fuchsia doesn't use dlsym-based interceptors.
return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl());
}

static bool PointerIsMine(const void *ptr) {
// Fuchsia doesn't use dlsym-based interceptors.
return !SANITIZER_FUCHSIA &&
UNLIKELY(internal_allocator()->FromPrimary(ptr));
}

static void *Allocate(uptr size_in_bytes) {
void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
CHECK(internal_allocator()->FromPrimary(ptr));
Details::OnAllocate(ptr,
internal_allocator()->GetActuallyAllocatedSize(ptr));
return ptr;
}

static void *Callocate(SIZE_T nmemb, SIZE_T size) {
void *ptr = InternalCalloc(nmemb, size);
CHECK(internal_allocator()->FromPrimary(ptr));
Details::OnAllocate(ptr,
internal_allocator()->GetActuallyAllocatedSize(ptr));
return ptr;
}

static void Free(void *ptr) {
uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
Details::OnFree(ptr, size);
InternalFree(ptr);
}

static void *Realloc(void *ptr, uptr new_size) {
if (!ptr)
return Allocate(new_size);
CHECK(internal_allocator()->FromPrimary(ptr));
if (!new_size) {
Free(ptr);
return nullptr;
}
uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
uptr memcpy_size = Min(new_size, size);
void *new_ptr = Allocate(new_size);
if (new_ptr)
internal_memcpy(new_ptr, ptr, memcpy_size);
Free(ptr);
return new_ptr;
}

static void OnAllocate(const void *ptr, uptr size) {}
static void OnFree(const void *ptr, uptr size) {}
};

} // namespace __sanitizer

#endif // SANITIZER_ALLOCATOR_DLSYM_H
20 changes: 9 additions & 11 deletions compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <sys/mman.h>

#include "interception/interception.h"
#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_mac.h"

// Similar code is used in Google Perftools,
Expand Down Expand Up @@ -192,20 +193,15 @@ void *__sanitizer_mz_malloc(malloc_zone_t *zone, uptr size) {
return p;
}

struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; }
};

extern "C"
SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) {
if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
const size_t kCallocPoolSize = 1024;
static uptr calloc_memory_for_dlsym[kCallocPoolSize];
static size_t allocated;
size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
void *mem = (void*)&calloc_memory_for_dlsym[allocated];
allocated += size_in_words;
CHECK(allocated < kCallocPoolSize);
return mem;
}
if (DlsymAlloc::Use())
return DlsymAlloc::Callocate(nmemb, size);
COMMON_MALLOC_CALLOC(nmemb, size);
return p;
}
Expand All @@ -223,6 +219,8 @@ extern "C"
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) {
if (!ptr) return;
if (DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Free(ptr);
COMMON_MALLOC_FREE(ptr);
}

Expand Down