135 changes: 50 additions & 85 deletions libc/src/__support/threads/linux/mutex.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,114 +9,79 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H

#include "hdr/types/pid_t.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/libc_assert.h"
#include "src/__support/threads/linux/futex_utils.h"
#include "src/__support/threads/linux/raw_mutex.h"
#include "src/__support/threads/mutex_common.h"

namespace LIBC_NAMESPACE {
struct Mutex {

// TODO: support shared/recursive/robust mutexes.
class Mutex final : private RawMutex {
// reserved timed, may be useful when combined with other flags.
unsigned char timed;
unsigned char recursive;
unsigned char robust;
unsigned char pshared;

void *owner;
// TLS address may not work across forked processes. Use thread id instead.
pid_t owner;
unsigned long long lock_count;

Futex futex_word;

enum class LockState : FutexWordType {
Free,
Locked,
Waiting,
};

public:
constexpr Mutex(bool istimed, bool isrecursive, bool isrobust)
: timed(istimed), recursive(isrecursive), robust(isrobust),
owner(nullptr), lock_count(0),
futex_word(FutexWordType(LockState::Free)) {}

static MutexError init(Mutex *mutex, bool istimed, bool isrecur,
bool isrobust) {
mutex->timed = istimed;
LIBC_INLINE constexpr Mutex(bool is_timed, bool is_recursive, bool is_robust,
bool is_pshared)
: RawMutex(), timed(is_timed), recursive(is_recursive), robust(is_robust),
pshared(is_pshared), owner(0), lock_count(0) {}

LIBC_INLINE static MutexError init(Mutex *mutex, bool is_timed, bool isrecur,
bool isrobust, bool is_pshared) {
RawMutex::init(mutex);
mutex->timed = is_timed;
mutex->recursive = isrecur;
mutex->robust = isrobust;
mutex->owner = nullptr;
mutex->pshared = is_pshared;
mutex->owner = 0;
mutex->lock_count = 0;
mutex->futex_word.set(FutexWordType(LockState::Free));
return MutexError::NONE;
}

static MutexError destroy(Mutex *) { return MutexError::NONE; }

MutexError reset();

MutexError lock() {
bool was_waiting = false;
while (true) {
FutexWordType mutex_status = FutexWordType(LockState::Free);
FutexWordType locked_status = FutexWordType(LockState::Locked);

if (futex_word.compare_exchange_strong(
mutex_status, FutexWordType(LockState::Locked))) {
if (was_waiting)
futex_word = FutexWordType(LockState::Waiting);
return MutexError::NONE;
}
LIBC_INLINE static MutexError destroy(Mutex *lock) {
LIBC_ASSERT(lock->owner == 0 && lock->lock_count == 0 &&
"Mutex destroyed while being locked.");
RawMutex::destroy(lock);
return MutexError::NONE;
}

switch (LockState(mutex_status)) {
case LockState::Waiting:
// If other threads are waiting already, then join them. Note that the
// futex syscall will block if the futex data is still
// `LockState::Waiting` (the 4th argument to the syscall function
// below.)
futex_word.wait(FutexWordType(LockState::Waiting));
was_waiting = true;
// Once woken up/unblocked, try everything all over.
continue;
case LockState::Locked:
// Mutex has been locked by another thread so set the status to
// LockState::Waiting.
if (futex_word.compare_exchange_strong(
locked_status, FutexWordType(LockState::Waiting))) {
// If we are able to set the futex data to `LockState::Waiting`, then
// we will wait for the futex to be woken up. Note again that the
// following syscall will block only if the futex data is still
// `LockState::Waiting`.
futex_word.wait(FutexWordType(LockState::Waiting));
was_waiting = true;
}
continue;
case LockState::Free:
// If it was LockState::Free, we shouldn't be here at all.
return MutexError::BAD_LOCK_STATE;
}
}
// TODO: record owner and lock count.
LIBC_INLINE MutexError lock() {
// Since timeout is not specified, we do not need to check the return value.
this->RawMutex::lock(
/* timeout=*/cpp::nullopt, this->pshared);
return MutexError::NONE;
}

MutexError unlock() {
while (true) {
FutexWordType mutex_status = FutexWordType(LockState::Waiting);
if (futex_word.compare_exchange_strong(mutex_status,
FutexWordType(LockState::Free))) {
// If any thread is waiting to be woken up, then do it.
futex_word.notify_one();
return MutexError::NONE;
}
// TODO: record owner and lock count.
LIBC_INLINE MutexError timed_lock(internal::AbsTimeout abs_time) {
if (this->RawMutex::lock(abs_time, this->pshared))
return MutexError::NONE;
return MutexError::TIMEOUT;
}

if (mutex_status == FutexWordType(LockState::Locked)) {
// If nobody was waiting at this point, just free it.
if (futex_word.compare_exchange_strong(mutex_status,
FutexWordType(LockState::Free)))
return MutexError::NONE;
} else {
// This can happen, for example if some thread tries to unlock an
// already free mutex.
return MutexError::UNLOCK_WITHOUT_LOCK;
}
}
LIBC_INLINE MutexError unlock() {
if (this->RawMutex::unlock(this->pshared))
return MutexError::NONE;
return MutexError::UNLOCK_WITHOUT_LOCK;
}

MutexError trylock();
// TODO: record owner and lock count.
LIBC_INLINE MutexError try_lock() {
if (this->RawMutex::try_lock())
return MutexError::NONE;
return MutexError::BUSY;
}
};

} // namespace LIBC_NAMESPACE
Expand Down
130 changes: 130 additions & 0 deletions libc/src/__support/threads/linux/raw_mutex.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
//===--- Implementation of a Linux RawMutex class ---------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H

#include "src/__support/CPP/optional.h"
#include "src/__support/common.h"
#include "src/__support/libc_assert.h"
#include "src/__support/macros/attributes.h"
#include "src/__support/threads/linux/futex_utils.h"
#include "src/__support/threads/linux/futex_word.h"
#include "src/__support/threads/sleep.h"
#include "src/__support/time/linux/abs_timeout.h"

#ifndef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
#define LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 1
#warning "LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY is not defined, defaulting to 1"
#endif

#if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
#include "src/__support/time/linux/monotonicity.h"
#endif

#ifndef LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT
#define LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT 100
#warning \
"LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT is not defined, defaulting to 100"
#endif

namespace LIBC_NAMESPACE {
// Lock is a simple timable lock for internal usage.
// This is separated from Mutex because this one does not need to consider
// robustness and reentrancy. Also, this one has spin optimization for shorter
// critical sections.
class RawMutex {
protected:
Futex futex;
LIBC_INLINE_VAR static constexpr FutexWordType UNLOCKED = 0b00;
LIBC_INLINE_VAR static constexpr FutexWordType LOCKED = 0b01;
LIBC_INLINE_VAR static constexpr FutexWordType IN_CONTENTION = 0b10;

private:
LIBC_INLINE FutexWordType spin(unsigned spin_count) {
FutexWordType result;
for (;;) {
result = futex.load(cpp::MemoryOrder::RELAXED);
// spin until one of the following conditions is met:
// - the mutex is unlocked
// - the mutex is in contention
// - the spin count reaches 0
if (result != LOCKED || spin_count == 0u)
return result;
// Pause the pipeline to avoid extraneous memory operations due to
// speculation.
sleep_briefly();
spin_count--;
};
}

// Return true if the lock is acquired. Return false if timeout happens before
// the lock is acquired.
LIBC_INLINE bool lock_slow(cpp::optional<Futex::Timeout> timeout,
bool is_pshared, unsigned spin_count) {
FutexWordType state = spin(spin_count);
// Before go into contention state, try to grab the lock.
if (state == UNLOCKED &&
futex.compare_exchange_strong(state, LOCKED, cpp::MemoryOrder::ACQUIRE,
cpp::MemoryOrder::RELAXED))
return true;
#if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
/* ADL should kick in */
if (timeout)
ensure_monotonicity(*timeout);
#endif
for (;;) {
// Try to grab the lock if it is unlocked. Mark the contention flag if it
// is locked.
if (state != IN_CONTENTION &&
futex.exchange(IN_CONTENTION, cpp::MemoryOrder::ACQUIRE) == UNLOCKED)
return true;
// Contention persists. Park the thread and wait for further notification.
if (ETIMEDOUT == -futex.wait(IN_CONTENTION, timeout, is_pshared))
return false;
// Continue to spin after waking up.
state = spin(spin_count);
}
}

LIBC_INLINE void wake(bool is_pshared) { futex.notify_one(is_pshared); }

public:
LIBC_INLINE static void init(RawMutex *mutex) { mutex->futex = UNLOCKED; }
LIBC_INLINE constexpr RawMutex() : futex(UNLOCKED) {}
[[nodiscard]] LIBC_INLINE bool try_lock() {
FutexWordType expected = UNLOCKED;
// Use strong version since this is a one-time operation.
return futex.compare_exchange_strong(
expected, LOCKED, cpp::MemoryOrder::ACQUIRE, cpp::MemoryOrder::RELAXED);
}
LIBC_INLINE bool
lock(cpp::optional<Futex::Timeout> timeout = cpp::nullopt,
bool is_shared = false,
unsigned spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) {
// Timeout will not be checked if immediate lock is possible.
if (LIBC_LIKELY(try_lock()))
return true;
return lock_slow(timeout, is_shared, spin_count);
}
LIBC_INLINE bool unlock(bool is_pshared = false) {
FutexWordType prev = futex.exchange(UNLOCKED, cpp::MemoryOrder::RELEASE);
// if there is someone waiting, wake them up
if (LIBC_UNLIKELY(prev == IN_CONTENTION))
wake(is_pshared);
// Detect invalid unlock operation.
return prev != UNLOCKED;
}
LIBC_INLINE void static destroy([[maybe_unused]] RawMutex *lock) {
LIBC_ASSERT(lock->futex == UNLOCKED && "Mutex destroyed while used.");
}
LIBC_INLINE Futex &get_raw_futex() { return futex; }
LIBC_INLINE void reset() { futex = UNLOCKED; }
};
} // namespace LIBC_NAMESPACE

#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
8 changes: 6 additions & 2 deletions libc/src/__support/threads/thread.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ class TSSKeyMgr {
cpp::array<TSSKeyUnit, TSS_KEY_COUNT> units;

public:
constexpr TSSKeyMgr() : mtx(false, false, false) {}
constexpr TSSKeyMgr()
: mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
/*pshared=*/false) {}

cpp::optional<unsigned int> new_key(TSSDtor *dtor) {
cpp::lock_guard lock(mtx);
Expand Down Expand Up @@ -111,7 +113,9 @@ class ThreadAtExitCallbackMgr {
FixedVector<AtExitUnit, 1024> callback_list;

public:
constexpr ThreadAtExitCallbackMgr() : mtx(false, false, false) {}
constexpr ThreadAtExitCallbackMgr()
: mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
/*pshared=*/false) {}

int add_callback(AtExitCallback *callback, void *obj) {
cpp::lock_guard lock(mtx);
Expand Down
5 changes: 3 additions & 2 deletions libc/src/pthread/pthread_mutex_init.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@ LLVM_LIBC_FUNCTION(int, pthread_mutex_init,
const pthread_mutexattr_t *__restrict attr)) {
auto mutexattr = attr == nullptr ? DEFAULT_MUTEXATTR : *attr;
auto err =
Mutex::init(reinterpret_cast<Mutex *>(m), false,
Mutex::init(reinterpret_cast<Mutex *>(m), /*is_timed=*/true,
get_mutexattr_type(mutexattr) & PTHREAD_MUTEX_RECURSIVE,
get_mutexattr_robust(mutexattr) & PTHREAD_MUTEX_ROBUST);
get_mutexattr_robust(mutexattr) & PTHREAD_MUTEX_ROBUST,
get_mutexattr_pshared(mutexattr) & PTHREAD_PROCESS_SHARED);
return err == MutexError::NONE ? 0 : EAGAIN;
}

Expand Down
5 changes: 5 additions & 0 deletions libc/src/pthread/pthread_mutexattr.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ LIBC_INLINE int get_mutexattr_robust(pthread_mutexattr_t attr) {
unsigned(PThreadMutexAttrPos::ROBUST_SHIFT);
}

LIBC_INLINE int get_mutexattr_pshared(pthread_mutexattr_t attr) {
return (attr & unsigned(PThreadMutexAttrPos::PSHARED_MASK)) >>
unsigned(PThreadMutexAttrPos::PSHARED_SHIFT);
}

} // namespace LIBC_NAMESPACE

#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_MUTEXATTR_H
3 changes: 2 additions & 1 deletion libc/src/stdlib/atexit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ namespace LIBC_NAMESPACE {

namespace {

Mutex handler_list_mtx(false, false, false);
Mutex handler_list_mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
/*pshared=*/false);

using AtExitCallback = void(void *);
using StdCAtExitCallback = void(void);
Expand Down
1 change: 1 addition & 0 deletions libc/src/threads/linux/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ add_header_library(
libc.src.__support.CPP.mutex
libc.src.__support.OSUtil.osutil
libc.src.__support.threads.mutex
libc.src.__support.threads.linux.raw_mutex
libc.src.__support.threads.linux.futex_utils
)

Expand Down
3 changes: 2 additions & 1 deletion libc/src/threads/mtx_init.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ static_assert(sizeof(Mutex) <= sizeof(mtx_t),

LLVM_LIBC_FUNCTION(int, mtx_init, (mtx_t * m, int type)) {
auto err = Mutex::init(reinterpret_cast<Mutex *>(m), type & mtx_timed,
type & mtx_recursive, 0);
type & mtx_recursive, /* is_robust */ false,
/* is_pshared */ false);
return err == MutexError::NONE ? thrd_success : thrd_error;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
#include "src/__support/threads/thread.h"
#include "test/IntegrationTest/test.h"

LIBC_NAMESPACE::Mutex mutex(false, false, false);
LIBC_NAMESPACE::Mutex mutex(/*timed=*/false, /*recursive=*/false,
/*robust=*/false, /*pshared=*/false);

int func(void *) {
mutex.lock();
Expand Down
1 change: 1 addition & 0 deletions libc/test/src/__support/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -207,3 +207,4 @@ add_subdirectory(FPUtil)
add_subdirectory(fixed_point)
add_subdirectory(HashTable)
add_subdirectory(time)
add_subdirectory(threads)
4 changes: 4 additions & 0 deletions libc/test/src/__support/threads/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
add_custom_target(libc-support-threads-tests)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
add_subdirectory(${LIBC_TARGET_OS})
endif()
12 changes: 12 additions & 0 deletions libc/test/src/__support/threads/linux/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
add_libc_test(
raw_mutex_test
SUITE
libc-support-threads-tests
SRCS
raw_mutex_test.cpp
DEPENDS
libc.src.__support.threads.linux.raw_mutex
libc.src.sys.mman.mmap
libc.src.sys.mman.munmap
libc.src.stdlib.exit
)
81 changes: 81 additions & 0 deletions libc/test/src/__support/threads/linux/raw_mutex_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
//===-- Unittests for Linux's RawMutex ------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "include/llvm-libc-macros/linux/time-macros.h"
#include "src/__support/CPP/atomic.h"
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/threads/linux/raw_mutex.h"
#include "src/__support/threads/sleep.h"
#include "src/__support/time/linux/clock_gettime.h"
#include "src/stdlib/exit.h"
#include "src/sys/mman/mmap.h"
#include "src/sys/mman/munmap.h"
#include "test/UnitTest/Test.h"
#include <sys/syscall.h>

TEST(LlvmLibcSupportThreadsRawMutexTest, SmokeTest) {
LIBC_NAMESPACE::RawMutex mutex;
ASSERT_TRUE(mutex.lock());
ASSERT_TRUE(mutex.unlock());
ASSERT_TRUE(mutex.try_lock());
ASSERT_FALSE(mutex.try_lock());
ASSERT_TRUE(mutex.unlock());
ASSERT_FALSE(mutex.unlock());
}

TEST(LlvmLibcSupportThreadsRawMutexTest, Timeout) {
LIBC_NAMESPACE::RawMutex mutex;
ASSERT_TRUE(mutex.lock());
timespec ts;
LIBC_NAMESPACE::internal::clock_gettime(CLOCK_MONOTONIC, &ts);
ts.tv_sec += 1;
// Timeout will be respected when deadlock happens.
auto timeout = LIBC_NAMESPACE::internal::AbsTimeout::from_timespec(ts, false);
ASSERT_TRUE(timeout.has_value());
// The following will timeout
ASSERT_FALSE(mutex.lock(*timeout));
ASSERT_TRUE(mutex.unlock());
// Test that the mutex works after the timeout.
ASSERT_TRUE(mutex.lock());
ASSERT_TRUE(mutex.unlock());
// If a lock can be acquired directly, expired timeout will not count.
// Notice that the timeout is already reached during preivous deadlock.
ASSERT_TRUE(mutex.lock(*timeout));
ASSERT_TRUE(mutex.unlock());
}

TEST(LlvmLibcSupportThreadsRawMutexTest, PSharedLock) {
struct SharedData {
LIBC_NAMESPACE::RawMutex mutex;
LIBC_NAMESPACE::cpp::Atomic<size_t> finished;
int data;
};
void *addr =
LIBC_NAMESPACE::mmap(nullptr, sizeof(SharedData), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
ASSERT_NE(addr, MAP_FAILED);
auto *shared = reinterpret_cast<SharedData *>(addr);
shared->data = 0;
LIBC_NAMESPACE::RawMutex::init(&shared->mutex);
// Avoid pull in our own implementation of pthread_t.
long pid = LIBC_NAMESPACE::syscall_impl<long>(SYS_fork);
for (int i = 0; i < 10000; ++i) {
shared->mutex.lock(LIBC_NAMESPACE::cpp::nullopt, true);
shared->data++;
shared->mutex.unlock(true);
}
// Mark the thread as finished.
shared->finished.fetch_add(1);
// let the child exit early to avoid output pollution
if (pid == 0)
LIBC_NAMESPACE::exit(0);
while (shared->finished.load() != 2)
LIBC_NAMESPACE::sleep_briefly();
ASSERT_EQ(shared->data, 20000);
LIBC_NAMESPACE::munmap(addr, sizeof(SharedData));
}