Skip to content

Commit

Permalink
[scudo] Scudo thread specific data refactor, part 1
Browse files Browse the repository at this point in the history
Summary:
We are going through an overhaul of Scudo's TSD, to allow for new platforms
to be integrated more easily, and make the code more sound.

This first part is mostly renaming, preferring some shorter names, correcting
some comments. I removed `getPrng` and `getAllocatorCache` to directly access
the members, there was not really any benefit to them (and it was suggested by
Dmitry in D37590).

The only functional change is in `scudo_tls_android.cpp`: we enforce bounds to
the `NumberOfTSDs` and most of the logic in `getTSDAndLockSlow` is skipped if we
only have 1 TSD.

Reviewers: alekseyshl, dvyukov, kcc

Reviewed By: dvyukov

Subscribers: llvm-commits, srhines

Differential Revision: https://reviews.llvm.org/D38139

llvm-svn: 313987
  • Loading branch information
Kostya Kortchinsky committed Sep 22, 2017
1 parent ae42181 commit 3924809
Show file tree
Hide file tree
Showing 8 changed files with 90 additions and 100 deletions.
60 changes: 24 additions & 36 deletions compiler-rt/lib/scudo/scudo_allocator.cpp
Expand Up @@ -250,19 +250,11 @@ struct QuarantineCallback {
typedef Quarantine<QuarantineCallback, ScudoChunk> ScudoQuarantine;
typedef ScudoQuarantine::Cache ScudoQuarantineCache;
COMPILER_CHECK(sizeof(ScudoQuarantineCache) <=
sizeof(ScudoThreadContext::QuarantineCachePlaceHolder));
sizeof(ScudoTSD::QuarantineCachePlaceHolder));

AllocatorCache *getAllocatorCache(ScudoThreadContext *ThreadContext) {
return &ThreadContext->Cache;
}

ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) {
return reinterpret_cast<
ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder);
}

ScudoPrng *getPrng(ScudoThreadContext *ThreadContext) {
return &ThreadContext->Prng;
ScudoQuarantineCache *getQuarantineCache(ScudoTSD *TSD) {
return reinterpret_cast<ScudoQuarantineCache *>(
TSD->QuarantineCachePlaceHolder);
}

struct ScudoAllocator {
Expand Down Expand Up @@ -381,12 +373,11 @@ struct ScudoAllocator {
uptr AllocSize;
if (FromPrimary) {
AllocSize = AlignedSize;
ScudoThreadContext *ThreadContext = getThreadContextAndLock();
if (LIKELY(ThreadContext)) {
Salt = getPrng(ThreadContext)->getU8();
Ptr = BackendAllocator.allocatePrimary(getAllocatorCache(ThreadContext),
AllocSize);
ThreadContext->unlock();
ScudoTSD *TSD = getTSDAndLock();
if (LIKELY(TSD)) {
Salt = TSD->Prng.getU8();
Ptr = BackendAllocator.allocatePrimary(&TSD->Cache, AllocSize);
TSD->unlock();
} else {
SpinMutexLock l(&FallbackMutex);
Salt = FallbackPrng.getU8();
Expand Down Expand Up @@ -454,11 +445,10 @@ struct ScudoAllocator {
Chunk->eraseHeader();
void *Ptr = Chunk->getAllocBeg(Header);
if (Header->FromPrimary) {
ScudoThreadContext *ThreadContext = getThreadContextAndLock();
if (LIKELY(ThreadContext)) {
getBackendAllocator().deallocatePrimary(
getAllocatorCache(ThreadContext), Ptr);
ThreadContext->unlock();
ScudoTSD *TSD = getTSDAndLock();
if (LIKELY(TSD)) {
getBackendAllocator().deallocatePrimary(&TSD->Cache, Ptr);
TSD->unlock();
} else {
SpinMutexLock Lock(&FallbackMutex);
getBackendAllocator().deallocatePrimary(&FallbackAllocatorCache, Ptr);
Expand All @@ -476,13 +466,12 @@ struct ScudoAllocator {
UnpackedHeader NewHeader = *Header;
NewHeader.State = ChunkQuarantine;
Chunk->compareExchangeHeader(&NewHeader, Header);
ScudoThreadContext *ThreadContext = getThreadContextAndLock();
if (LIKELY(ThreadContext)) {
AllocatorQuarantine.Put(getQuarantineCache(ThreadContext),
QuarantineCallback(
getAllocatorCache(ThreadContext)),
ScudoTSD *TSD = getTSDAndLock();
if (LIKELY(TSD)) {
AllocatorQuarantine.Put(getQuarantineCache(TSD),
QuarantineCallback(&TSD->Cache),
Chunk, EstimatedSize);
ThreadContext->unlock();
TSD->unlock();
} else {
SpinMutexLock l(&FallbackMutex);
AllocatorQuarantine.Put(&FallbackQuarantineCache,
Expand Down Expand Up @@ -607,11 +596,10 @@ struct ScudoAllocator {
return allocate(NMemB * Size, MinAlignment, FromMalloc, true);
}

void commitBack(ScudoThreadContext *ThreadContext) {
AllocatorCache *Cache = getAllocatorCache(ThreadContext);
AllocatorQuarantine.Drain(getQuarantineCache(ThreadContext),
QuarantineCallback(Cache));
BackendAllocator.destroyCache(Cache);
void commitBack(ScudoTSD *TSD) {
AllocatorQuarantine.Drain(getQuarantineCache(TSD),
QuarantineCallback(&TSD->Cache));
BackendAllocator.destroyCache(&TSD->Cache);
}

uptr getStats(AllocatorStat StatType) {
Expand All @@ -637,13 +625,13 @@ static void initScudoInternal(const AllocatorOptions &Options) {
Instance.init(Options);
}

void ScudoThreadContext::init() {
void ScudoTSD::init() {
getBackendAllocator().initCache(&Cache);
Prng.init();
memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
}

void ScudoThreadContext::commitBack() {
void ScudoTSD::commitBack() {
Instance.commitBack(this);
}

Expand Down
4 changes: 2 additions & 2 deletions compiler-rt/lib/scudo/scudo_tls.h
Expand Up @@ -28,7 +28,7 @@ namespace __scudo {
#include "scudo_tls_context_android.inc"
#include "scudo_tls_context_linux.inc"

struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform {
struct ALIGNED(64) ScudoTSD : public ScudoTSDPlatform {
AllocatorCache Cache;
ScudoPrng Prng;
uptr QuarantineCachePlaceHolder[4];
Expand All @@ -38,7 +38,7 @@ struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform {

void initThread(bool MinimalInit);

// Platform specific dastpath functions definitions.
// Platform specific fastpath functions definitions.
#include "scudo_tls_android.inc"
#include "scudo_tls_linux.inc"

Expand Down
81 changes: 42 additions & 39 deletions compiler-rt/lib/scudo/scudo_tls_android.cpp
Expand Up @@ -24,9 +24,9 @@ namespace __scudo {
static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
static pthread_key_t PThreadKey;

static atomic_uint32_t ThreadContextCurrentIndex;
static ScudoThreadContext *ThreadContexts;
static uptr NumberOfContexts;
static atomic_uint32_t CurrentIndex;
static ScudoTSD *TSDs;
static u32 NumberOfTSDs;

// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used as they allocate memory.
static uptr getNumberOfCPUs() {
Expand All @@ -42,52 +42,55 @@ static void initOnce() {
// TODO(kostyak): remove and restrict to N and above.
CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0);
initScudo();
NumberOfContexts = getNumberOfCPUs();
ThreadContexts = reinterpret_cast<ScudoThreadContext *>(
MmapOrDie(sizeof(ScudoThreadContext) * NumberOfContexts, __func__));
for (uptr i = 0; i < NumberOfContexts; i++)
ThreadContexts[i].init();
NumberOfTSDs = getNumberOfCPUs();
if (NumberOfTSDs == 0)
NumberOfTSDs = 1;
if (NumberOfTSDs > 32)
NumberOfTSDs = 32;
TSDs = reinterpret_cast<ScudoTSD *>(
MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs"));
for (u32 i = 0; i < NumberOfTSDs; i++)
TSDs[i].init();
}

void initThread(bool MinimalInit) {
pthread_once(&GlobalInitialized, initOnce);
// Initial context assignment is done in a plain round-robin fashion.
u32 Index = atomic_fetch_add(&ThreadContextCurrentIndex, 1,
memory_order_relaxed);
ScudoThreadContext *ThreadContext =
&ThreadContexts[Index % NumberOfContexts];
*get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed);
ScudoTSD *TSD = &TSDs[Index % NumberOfTSDs];
*get_android_tls_ptr() = reinterpret_cast<uptr>(TSD);
}

ScudoThreadContext *getThreadContextAndLockSlow() {
ScudoThreadContext *ThreadContext;
// Go through all the contexts and find the first unlocked one.
for (u32 i = 0; i < NumberOfContexts; i++) {
ThreadContext = &ThreadContexts[i];
if (ThreadContext->tryLock()) {
*get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
return ThreadContext;
ScudoTSD *getTSDAndLockSlow() {
ScudoTSD *TSD;
if (NumberOfTSDs > 1) {
// Go through all the contexts and find the first unlocked one.
for (u32 i = 0; i < NumberOfTSDs; i++) {
TSD = &TSDs[i];
if (TSD->tryLock()) {
*get_android_tls_ptr() = reinterpret_cast<uptr>(TSD);
return TSD;
}
}
}
// No luck, find the one with the lowest precedence, and slow lock it.
u64 Precedence = UINT64_MAX;
for (u32 i = 0; i < NumberOfContexts; i++) {
u64 SlowLockPrecedence = ThreadContexts[i].getSlowLockPrecedence();
if (SlowLockPrecedence && SlowLockPrecedence < Precedence) {
ThreadContext = &ThreadContexts[i];
Precedence = SlowLockPrecedence;
// No luck, find the one with the lowest Precedence, and slow lock it.
u64 LowestPrecedence = UINT64_MAX;
for (u32 i = 0; i < NumberOfTSDs; i++) {
u64 Precedence = TSDs[i].getPrecedence();
if (Precedence && Precedence < LowestPrecedence) {
TSD = &TSDs[i];
LowestPrecedence = Precedence;
}
}
if (LIKELY(LowestPrecedence != UINT64_MAX)) {
TSD->lock();
*get_android_tls_ptr() = reinterpret_cast<uptr>(TSD);
return TSD;
}
}
if (LIKELY(Precedence != UINT64_MAX)) {
ThreadContext->lock();
*get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
return ThreadContext;
}
// Last resort (can this happen?), stick with the current one.
ThreadContext =
reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr());
ThreadContext->lock();
return ThreadContext;
// Last resort, stick with the current one.
TSD = reinterpret_cast<ScudoTSD *>(*get_android_tls_ptr());
TSD->lock();
return TSD;
}

} // namespace __scudo
Expand Down
15 changes: 7 additions & 8 deletions compiler-rt/lib/scudo/scudo_tls_android.inc
Expand Up @@ -26,17 +26,16 @@ ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
initThread(MinimalInit);
}

ScudoThreadContext *getThreadContextAndLockSlow();
ScudoTSD *getTSDAndLockSlow();

ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() {
ScudoThreadContext *ThreadContext =
reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr());
CHECK(ThreadContext);
ALWAYS_INLINE ScudoTSD *getTSDAndLock() {
ScudoTSD *TSD = reinterpret_cast<ScudoTSD *>(*get_android_tls_ptr());
CHECK(TSD);
// Try to lock the currently associated context.
if (ThreadContext->tryLock())
return ThreadContext;
if (TSD->tryLock())
return TSD;
// If it failed, go the slow path.
return getThreadContextAndLockSlow();
return getTSDAndLockSlow();
}

#endif // SANITIZER_LINUX && SANITIZER_ANDROID
Expand Down
16 changes: 8 additions & 8 deletions compiler-rt/lib/scudo/scudo_tls_context_android.inc
Expand Up @@ -20,33 +20,33 @@

#if SANITIZER_LINUX && SANITIZER_ANDROID

struct ScudoThreadContextPlatform {
struct ScudoTSDPlatform {
INLINE bool tryLock() {
if (Mutex.TryLock()) {
atomic_store_relaxed(&SlowLockPrecedence, 0);
atomic_store_relaxed(&Precedence, 0);
return true;
}
if (atomic_load_relaxed(&SlowLockPrecedence) == 0)
atomic_store_relaxed(&SlowLockPrecedence, NanoTime());
if (atomic_load_relaxed(&Precedence) == 0)
atomic_store_relaxed(&Precedence, NanoTime());
return false;
}

INLINE void lock() {
Mutex.Lock();
atomic_store_relaxed(&SlowLockPrecedence, 0);
atomic_store_relaxed(&Precedence, 0);
}

INLINE void unlock() {
Mutex.Unlock();
}

INLINE u64 getSlowLockPrecedence() {
return atomic_load_relaxed(&SlowLockPrecedence);
INLINE u64 getPrecedence() {
return atomic_load_relaxed(&Precedence);
}

private:
StaticSpinMutex Mutex;
atomic_uint64_t SlowLockPrecedence;
atomic_uint64_t Precedence;
};

#endif // SANITIZER_LINUX && SANITIZER_ANDROID
Expand Down
2 changes: 1 addition & 1 deletion compiler-rt/lib/scudo/scudo_tls_context_linux.inc
Expand Up @@ -20,7 +20,7 @@

#if SANITIZER_LINUX && !SANITIZER_ANDROID

struct ScudoThreadContextPlatform {
struct ScudoTSDPlatform {
ALWAYS_INLINE void unlock() {}
};

Expand Down
6 changes: 3 additions & 3 deletions compiler-rt/lib/scudo/scudo_tls_linux.cpp
Expand Up @@ -28,7 +28,7 @@ static pthread_key_t PThreadKey;
__attribute__((tls_model("initial-exec")))
THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized;
__attribute__((tls_model("initial-exec")))
THREADLOCAL ScudoThreadContext ThreadLocalContext;
THREADLOCAL ScudoTSD TSD;

static void teardownThread(void *Ptr) {
uptr I = reinterpret_cast<uptr>(Ptr);
Expand All @@ -43,7 +43,7 @@ static void teardownThread(void *Ptr) {
reinterpret_cast<void *>(I - 1)) == 0))
return;
}
ThreadLocalContext.commitBack();
TSD.commitBack();
ScudoThreadState = ThreadTornDown;
}

Expand All @@ -59,7 +59,7 @@ void initThread(bool MinimalInit) {
return;
CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(
GetPthreadDestructorIterations())), 0);
ThreadLocalContext.init();
TSD.init();
ScudoThreadState = ThreadInitialized;
}

Expand Down
6 changes: 3 additions & 3 deletions compiler-rt/lib/scudo/scudo_tls_linux.inc
Expand Up @@ -29,18 +29,18 @@ enum ThreadState : u8 {
__attribute__((tls_model("initial-exec")))
extern THREADLOCAL ThreadState ScudoThreadState;
__attribute__((tls_model("initial-exec")))
extern THREADLOCAL ScudoThreadContext ThreadLocalContext;
extern THREADLOCAL ScudoTSD TSD;

ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
if (LIKELY(ScudoThreadState != ThreadNotInitialized))
return;
initThread(MinimalInit);
}

ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() {
ALWAYS_INLINE ScudoTSD *getTSDAndLock() {
if (UNLIKELY(ScudoThreadState != ThreadInitialized))
return nullptr;
return &ThreadLocalContext;
return &TSD;
}

#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
Expand Down

0 comments on commit 3924809

Please sign in to comment.