diff --git a/libc/include/llvm-libc-macros/pthread-macros.h b/libc/include/llvm-libc-macros/pthread-macros.h index d6518189f1ccb..e23d2873e0891 100644 --- a/libc/include/llvm-libc-macros/pthread-macros.h +++ b/libc/include/llvm-libc-macros/pthread-macros.h @@ -32,18 +32,24 @@ #ifdef __linux__ #define PTHREAD_MUTEX_INITIALIZER \ { \ - /* .__ftxw = */ {0}, /* .__priority_inherit = */ 0, \ - /* .__recursive = */ 0, /* .__robust = */ 0, \ - /* .__pshared = */ 0, /* .__error_checking = */ 0, \ - /* .__owner = */ 0, /* .__lock_count = */ 0, \ + /* .__ftxw = */ {0}, \ + /* .__priority_inherit = */ 0, \ + /* .__recursive = */ 0, \ + /* .__robust = */ 0, \ + /* .__pshared = */ 0, \ + /* .__error_checking = */ 0, \ + {/* .__owner = */ 0, /* .__lock_count = */ 0}, \ } #else #define PTHREAD_MUTEX_INITIALIZER \ { \ - /* .__ftxw = */ {0}, /* .__priority_inherit = */ 0, \ - /* .__recursive = */ 0, /* .__robust = */ 0, \ - /* .__pshared = */ 0, /* .__error_checking = */ 0, \ - /* .__owner = */ 0, /* .__lock_count = */ 0, \ + /* .__ftxw = */ {0}, \ + /* .__priority_inherit = */ 0, \ + /* .__recursive = */ 0, \ + /* .__robust = */ 0, \ + /* .__pshared = */ 0, \ + /* .__error_checking = */ 0, \ + {/* .__owner = */ 0, /* .__lock_count = */ 0}, \ } #endif diff --git a/libc/include/llvm-libc-types/__mutex_type.h b/libc/include/llvm-libc-types/__mutex_type.h index 40f37c5235f2e..adc379318e675 100644 --- a/libc/include/llvm-libc-types/__mutex_type.h +++ b/libc/include/llvm-libc-types/__mutex_type.h @@ -26,8 +26,10 @@ typedef struct { unsigned int __pshared : 1; unsigned int __error_checking : 1; - pid_t __owner; - size_t __lock_count; + struct { + pid_t __owner; + size_t __lock_count; + }; } __mutex_type; #endif // LLVM_LIBC_TYPES___MUTEX_TYPE_H diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt index 0846a78bbf904..560235b0bad19 100644 --- a/libc/src/__support/threads/CMakeLists.txt +++ b/libc/src/__support/threads/CMakeLists.txt @@ -69,12 +69,18 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.futex_utils) libc.src.__support.threads.identifier ) + set(pi_mutex_dep) + if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.pi_mutex) + set(pi_mutex_dep .${LIBC_TARGET_OS}.pi_mutex) + endif() + add_header_library( unix_mutex HDRS unix_mutex.h DEPENDS .raw_mutex + ${pi_mutex_dep} libc.src.__support.CPP.atomic ) diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt index 8ce19634c41b1..2e1e701f9ec84 100644 --- a/libc/src/__support/threads/linux/CMakeLists.txt +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -24,6 +24,22 @@ add_header_library( libc.src.__support.time.abs_timeout ) +add_header_library( + pi_mutex + HDRS + pi_mutex.h + DEPENDS + .futex_utils + .futex_word_type + libc.hdr.errno_macros + libc.src.__support.OSUtil.osutil + libc.src.__support.CPP.limits + libc.src.__support.CPP.optional + libc.src.__support.time.monotonicity + libc.src.__support.macros.attributes + libc.src.__support.macros.config +) + add_object_library( thread SRCS diff --git a/libc/src/__support/threads/linux/pi_mutex.h b/libc/src/__support/threads/linux/pi_mutex.h new file mode 100644 index 0000000000000..bcabbd01f95ac --- /dev/null +++ b/libc/src/__support/threads/linux/pi_mutex.h @@ -0,0 +1,175 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Linux priority inheritance mutex support. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_PI_MUTEX_H +#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_PI_MUTEX_H + +#include "hdr/errno_macros.h" +#include "src/__support/CPP/limits.h" +#include "src/__support/CPP/new.h" +#include "src/__support/CPP/optional.h" +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/macros/attributes.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" +#include "src/__support/threads/identifier.h" +#include "src/__support/threads/linux/futex_utils.h" +#include "src/__support/threads/mutex_common.h" + +#include + +#ifdef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY +#include "src/__support/time/monotonicity.h" +#endif + +namespace LIBC_NAMESPACE_DECL { + +class PIMutex { +protected: + Futex owner; + // Number of recursive locks minus one. + size_t recursive_count; + +public: + enum class Type { Normal, ErrorChecking, Recursive }; + LIBC_INLINE static void init(PIMutex *mutex) { + mutex->owner.store(0); + mutex->recursive_count = 0; + } + LIBC_INLINE constexpr PIMutex() : owner(0), recursive_count(0) {} + LIBC_INLINE MutexError try_lock(Type type) { + FutexWordType old_owner = 0; + auto current = static_cast(internal::gettid()); + if (owner.compare_exchange_strong(old_owner, current, + cpp::MemoryOrder::ACQUIRE, + cpp::MemoryOrder::RELAXED)) + return MutexError::NONE; + + if (current == (old_owner & FUTEX_TID_MASK)) { + switch (type) { + case Type::Normal: + break; + case Type::ErrorChecking: + return MutexError::DEADLOCK; + case Type::Recursive: + if (LIBC_UNLIKELY(recursive_count == + cpp::numeric_limits::max())) + return MutexError::OVERFLOW; + recursive_count++; + return MutexError::NONE; + } + } + + return MutexError::BUSY; + } + LIBC_INLINE MutexError + lock(Type type, cpp::optional timeout = cpp::nullopt, + bool is_shared = false) { + MutexError result = try_lock(type); + if (result != MutexError::BUSY) + return result; + +#ifdef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY + if (timeout) + ensure_monotonicity(*timeout); +#endif + + int op = is_shared ? FUTEX_LOCK_PI : FUTEX_LOCK_PI_PRIVATE; + for (;;) { + ErrorOr ret = linux_syscalls::syscall_checked( + /*syscall_number=*/FUTEX_SYSCALL_ID, + /*futex_addr=*/&owner, + /*op=*/op, + /*ignored=*/0, + /*timeout=*/timeout ? &timeout->get_timespec() : nullptr, + /*ignored=*/nullptr, + /*ignored=*/0); + + if (ret.has_value()) + return MutexError::NONE; + + switch (ret.error()) { + case EINTR: + continue; + case ETIMEDOUT: + return MutexError::TIMEOUT; + case EDEADLK: + return MutexError::DEADLOCK; + default: + return MutexError::BAD_LOCK_STATE; + } + } + } + LIBC_INLINE MutexError unlock(Type type, bool is_shared) { + FutexWordType current = static_cast(internal::gettid()); + FutexWordType old_owner = current; + + if (LIBC_LIKELY(type == Type::Normal)) { + if (LIBC_LIKELY(owner.compare_exchange_strong(old_owner, 0, + cpp::MemoryOrder::RELEASE, + cpp::MemoryOrder::RELAXED))) + return MutexError::NONE; + } else { + old_owner = owner.load(cpp::MemoryOrder::RELAXED); + } + + if (current != (old_owner & FUTEX_TID_MASK)) + return MutexError::UNLOCK_WITHOUT_LOCK; + + if (type == Type::Recursive && recursive_count != 0) { + recursive_count--; + return MutexError::NONE; + } + + if (old_owner == current && LIBC_LIKELY(owner.compare_exchange_strong( + old_owner, 0, cpp::MemoryOrder::RELEASE, + cpp::MemoryOrder::RELAXED))) + return MutexError::NONE; + + int op = is_shared ? FUTEX_UNLOCK_PI : FUTEX_UNLOCK_PI_PRIVATE; + ErrorOr ret = linux_syscalls::syscall_checked( + /*syscall_number=*/FUTEX_SYSCALL_ID, + /*futex_addr=*/&owner, + /*op=*/op, + /*ignored=*/0, + /*ignored=*/nullptr, + /*ignored=*/nullptr, + /*ignored=*/0); + + if (ret.has_value()) + return MutexError::NONE; + + switch (ret.error()) { + case EPERM: + return MutexError::UNLOCK_WITHOUT_LOCK; + default: + return MutexError::BAD_LOCK_STATE; + } + } + LIBC_INLINE static MutexError destroy(PIMutex *lock) { + FutexWordType old_owner = 0; + if (lock->owner.compare_exchange_strong(old_owner, 0xffffffff, + cpp::MemoryOrder::RELAXED, + cpp::MemoryOrder::RELAXED)) + return MutexError::NONE; + return MutexError::BUSY; + } + LIBC_INLINE void reset() { + owner.store(0); + recursive_count = 0; + } +}; + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_PI_MUTEX_H diff --git a/libc/src/__support/threads/unix_mutex.h b/libc/src/__support/threads/unix_mutex.h index 272a1dfe6dbba..9afc3d337db2d 100644 --- a/libc/src/__support/threads/unix_mutex.h +++ b/libc/src/__support/threads/unix_mutex.h @@ -20,6 +20,10 @@ #include "src/__support/threads/mutex_common.h" #include "src/__support/threads/raw_mutex.h" +#ifdef LIBC_TARGET_OS_IS_LINUX +#include "src/__support/threads/linux/pi_mutex.h" +#endif + namespace LIBC_NAMESPACE_DECL { // TODO: support shared/recursive/robust mutexes. @@ -34,12 +38,31 @@ class Mutex final : private RawMutex { LIBC_PREFERED_TYPE(bool) unsigned int error_checking : 1; // TLS address may not work across forked processes. Use thread id instead. - cpp::Atomic owner; - size_t lock_count; + union { + struct { + cpp::Atomic owner; + size_t lock_count; + }; +#ifdef LIBC_TARGET_OS_IS_LINUX + // Special case, when "priority_inherit" is set, we ignore the base mutex + // and use this field. + PIMutex pi_mutex; +#endif + }; // CndVar needs to access Mutex as RawMutex friend class CndVar; +#ifdef LIBC_TARGET_OS_IS_LINUX + LIBC_INLINE PIMutex::Type pi_mutex_type() const { + if (is_recursive()) + return PIMutex::Type::Recursive; + if (is_error_checking()) + return PIMutex::Type::ErrorChecking; + return PIMutex::Type::Normal; + } +#endif + template LIBC_INLINE MutexError lock_impl(LockRoutine do_lock) { if (is_recursive() && owner == internal::gettid()) { @@ -70,9 +93,18 @@ class Mutex final : private RawMutex { bool is_error_checking = false) : RawMutex(), priority_inherit(is_priority_inherit), recursive(is_recursive), robust(is_robust), pshared(is_pshared), - error_checking(is_error_checking), owner(0), lock_count(0) {} + error_checking(is_error_checking), owner(0), lock_count(0) { +#ifdef LIBC_TARGET_OS_IS_LINUX + if (is_priority_inherit) + new (&pi_mutex) PIMutex{}; +#endif + } LIBC_INLINE static MutexError destroy(Mutex *lock) { +#ifdef LIBC_TARGET_OS_IS_LINUX + if (lock->priority_inherit) + return PIMutex::destroy(&lock->pi_mutex); +#endif LIBC_ASSERT(lock->owner == 0 && lock->lock_count == 0 && "Mutex destroyed while being locked."); RawMutex::destroy(lock); @@ -80,6 +112,11 @@ class Mutex final : private RawMutex { } LIBC_INLINE MutexError lock() { +#ifdef LIBC_TARGET_OS_IS_LINUX + if (priority_inherit) + return pi_mutex.lock(pi_mutex_type(), /*timeout=*/cpp::nullopt, + this->pshared); +#endif return lock_impl([this] { // Since timeout is not specified, we do not need to check the return // value. @@ -90,6 +127,10 @@ class Mutex final : private RawMutex { } LIBC_INLINE MutexError timed_lock(internal::AbsTimeout abs_time) { +#ifdef LIBC_TARGET_OS_IS_LINUX + if (priority_inherit) + return pi_mutex.lock(pi_mutex_type(), abs_time, this->pshared); +#endif return lock_impl([this, abs_time] { // TODO: check deadlock? POSIX made it optional. if (this->RawMutex::lock(abs_time, this->pshared)) @@ -99,6 +140,10 @@ class Mutex final : private RawMutex { } LIBC_INLINE MutexError unlock() { +#ifdef LIBC_TARGET_OS_IS_LINUX + if (priority_inherit) + return pi_mutex.unlock(pi_mutex_type(), this->pshared); +#endif if (is_recursive()) { // lock_count == 0 can happen if previous unlock is // suspended before signal frame @@ -121,6 +166,10 @@ class Mutex final : private RawMutex { } LIBC_INLINE MutexError try_lock() { +#ifdef LIBC_TARGET_OS_IS_LINUX + if (priority_inherit) + return pi_mutex.try_lock(pi_mutex_type()); +#endif return lock_impl([this] { if (this->RawMutex::try_lock()) return MutexError::NONE; diff --git a/libc/src/pthread/pthread_mutex_init.cpp b/libc/src/pthread/pthread_mutex_init.cpp index 73aad9d13792a..c44b585e89b83 100644 --- a/libc/src/pthread/pthread_mutex_init.cpp +++ b/libc/src/pthread/pthread_mutex_init.cpp @@ -45,9 +45,10 @@ LLVM_LIBC_FUNCTION(int, pthread_mutex_init, is_robust = true; bool is_pshared = get_mutexattr_pshared(mutexattr) == PTHREAD_PROCESS_SHARED; + bool is_priority_inherit = get_mutexattr_priority_inherit(mutexattr); - new (m) Mutex(/*is_priority_inherit=*/false, is_recursive, is_robust, - is_pshared, is_error_checking); + new (m) Mutex(is_priority_inherit, is_recursive, is_robust, is_pshared, + is_error_checking); return 0; } diff --git a/libc/src/pthread/pthread_mutexattr.h b/libc/src/pthread/pthread_mutexattr.h index be719b9d14997..2becdc34abc6f 100644 --- a/libc/src/pthread/pthread_mutexattr.h +++ b/libc/src/pthread/pthread_mutexattr.h @@ -26,7 +26,10 @@ enum class PThreadMutexAttrPos : unsigned int { PSHARED_SHIFT = 3, PSHARED_MASK = 0x1 << PSHARED_SHIFT, - // TODO: Add a mask for protocol and prioceiling when it is supported. + PRIORITY_INHERIT_SHIFT = 4, + PRIORITY_INHERIT_MASK = 0x1 << PRIORITY_INHERIT_SHIFT, + + // TODO: Add a mask for prioceiling when it is supported. }; constexpr pthread_mutexattr_t DEFAULT_MUTEXATTR = @@ -49,6 +52,10 @@ LIBC_INLINE int get_mutexattr_pshared(pthread_mutexattr_t attr) { unsigned(PThreadMutexAttrPos::PSHARED_SHIFT); } +LIBC_INLINE bool get_mutexattr_priority_inherit(pthread_mutexattr_t attr) { + return (attr & unsigned(PThreadMutexAttrPos::PRIORITY_INHERIT_MASK)) != 0; +} + } // namespace LIBC_NAMESPACE_DECL #endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_MUTEXATTR_H diff --git a/libc/test/integration/src/pthread/pthread_mutex_test.cpp b/libc/test/integration/src/pthread/pthread_mutex_test.cpp index 1a27fec139f98..f286864118789 100644 --- a/libc/test/integration/src/pthread/pthread_mutex_test.cpp +++ b/libc/test/integration/src/pthread/pthread_mutex_test.cpp @@ -15,6 +15,7 @@ #include "src/pthread/pthread_mutex_lock.h" #include "src/pthread/pthread_mutex_trylock.h" #include "src/pthread/pthread_mutex_unlock.h" +#include "src/pthread/pthread_mutexattr.h" #include "src/pthread/pthread_mutexattr_destroy.h" #include "src/pthread/pthread_mutexattr_init.h" #include "src/pthread/pthread_mutexattr_settype.h" @@ -35,6 +36,23 @@ static pthread_mutex_t snapshot_mutex(const void *mutex_storage) { return snapshot; } +static void set_priority_inherit(pthread_mutexattr_t *attr, + bool priority_inherit) { + if (priority_inherit) + *attr |= + unsigned(LIBC_NAMESPACE::PThreadMutexAttrPos::PRIORITY_INHERIT_MASK); +} + +static void init_mutex(pthread_mutex_t *mutex, bool priority_inherit) { + pthread_mutexattr_t attr; + ASSERT_EQ(LIBC_NAMESPACE::pthread_mutexattr_init(&attr), 0); + set_priority_inherit(&attr, priority_inherit); + ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_init(mutex, &attr), 0); + ASSERT_EQ(LIBC_NAMESPACE::pthread_mutexattr_destroy(&attr), 0); + pthread_mutex_t snapshot = snapshot_mutex(mutex); + ASSERT_EQ(bool(snapshot.__priority_inherit), priority_inherit); +} + pthread_mutex_t mutex; static int shared_int = START; @@ -53,8 +71,9 @@ void *counter([[maybe_unused]] void *arg) { return nullptr; } -void relay_counter() { - ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_init(&mutex, nullptr), 0); +void relay_counter(bool priority_inherit) { + shared_int = START; + init_mutex(&mutex, priority_inherit); // The idea of this test is that two competing threads will update // a counter only if the other thread has updated it. @@ -98,9 +117,9 @@ void *stepper([[maybe_unused]] void *arg) { return nullptr; } -void wait_and_step() { - ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_init(&start_lock, nullptr), 0); - ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_init(&step_lock, nullptr), 0); +void wait_and_step(bool priority_inherit) { + init_mutex(&start_lock, priority_inherit); + init_mutex(&step_lock, priority_inherit); // In this test, we start a new thread but block it before it can make a // step. Once we ensure that the thread is blocked, we unblock it. @@ -143,9 +162,9 @@ void wait_and_step() { LIBC_NAMESPACE::pthread_mutex_destroy(&step_lock); } -void trylock_test() { +void trylock_test(bool priority_inherit) { pthread_mutex_t trylock_mutex; - ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_init(&trylock_mutex, nullptr), 0); + init_mutex(&trylock_mutex, priority_inherit); ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_trylock(&trylock_mutex), 0); ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_trylock(&trylock_mutex), EBUSY); @@ -164,13 +183,14 @@ void *trylock_other_thread(void *arg) { return reinterpret_cast(uintptr_t(result)); } -void recursive_mutex_test() { +void recursive_mutex_test(bool priority_inherit) { pthread_mutexattr_t attr; pthread_mutex_t recursive_mutex; ASSERT_EQ(LIBC_NAMESPACE::pthread_mutexattr_init(&attr), 0); ASSERT_EQ( LIBC_NAMESPACE::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), 0); + set_priority_inherit(&attr, priority_inherit); ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_init(&recursive_mutex, &attr), 0); ASSERT_EQ(LIBC_NAMESPACE::pthread_mutexattr_destroy(&attr), 0); @@ -238,13 +258,14 @@ void initializer_acts_the_same_as_null_attr() { ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_destroy(&mutex_from_init), 0); } -void error_checking_mutex_test() { +void error_checking_mutex_test(bool priority_inherit) { pthread_mutexattr_t attr; pthread_mutex_t error_checking_mutex; ASSERT_EQ(LIBC_NAMESPACE::pthread_mutexattr_init(&attr), 0); ASSERT_EQ(LIBC_NAMESPACE::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK), 0); + set_priority_inherit(&attr, priority_inherit); ASSERT_EQ(LIBC_NAMESPACE::pthread_mutex_init(&error_checking_mutex, &attr), 0); ASSERT_EQ(LIBC_NAMESPACE::pthread_mutexattr_destroy(&attr), 0); @@ -288,9 +309,10 @@ void *waiter_func(void *) { return nullptr; } -void multiple_waiters() { - LIBC_NAMESPACE::pthread_mutex_init(&multiple_waiter_lock, nullptr); - LIBC_NAMESPACE::pthread_mutex_init(&counter_lock, nullptr); +void multiple_waiters(bool priority_inherit) { + wait_count = 0; + init_mutex(&multiple_waiter_lock, priority_inherit); + init_mutex(&counter_lock, priority_inherit); LIBC_NAMESPACE::pthread_mutex_lock(&multiple_waiter_lock); pthread_t waiters[THREAD_COUNT]; @@ -323,12 +345,15 @@ void multiple_waiters() { } TEST_MAIN() { - relay_counter(); - wait_and_step(); - trylock_test(); - recursive_mutex_test(); + for (int i = 0; i < 2; ++i) { + bool priority_inherit = i & 1; + relay_counter(priority_inherit); + wait_and_step(priority_inherit); + trylock_test(priority_inherit); + recursive_mutex_test(priority_inherit); + error_checking_mutex_test(priority_inherit); + multiple_waiters(priority_inherit); + } initializer_acts_the_same_as_null_attr(); - error_checking_mutex_test(); - multiple_waiters(); return 0; }