diff --git a/libc/config/linux/app.h b/libc/config/linux/app.h index f3d11da9fc14c..11ac6ee51e5a0 100644 --- a/libc/config/linux/app.h +++ b/libc/config/linux/app.h @@ -35,17 +35,6 @@ struct TLSImage { uintptr_t align; }; -// Linux manpage on `proc(5)` says that the aux vector is an array of -// unsigned long pairs. -// (see: https://man7.org/linux/man-pages/man5/proc.5.html) -using AuxEntryType = unsigned long; -// Using the naming convention from `proc(5)`. -// TODO: Would be nice to use the aux entry structure from elf.h when available. -struct AuxEntry { - AuxEntryType id; - AuxEntryType value; -}; - struct Args { uintptr_t argc; @@ -70,9 +59,6 @@ struct AppProperties { // Environment data. uintptr_t *env_ptr; - - // Auxiliary vector data. - AuxEntry *auxv_ptr; }; [[gnu::weak]] extern AppProperties app; diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt index f303e54ce7b3b..f6377ca9ff5a2 100644 --- a/libc/src/__support/OSUtil/linux/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt @@ -24,6 +24,18 @@ add_object_library( libc.include.sys_syscall ) +add_header_library( + auxv + HDRS + auxv.h + DEPENDS + libc.hdr.fcntl_macros + libc.src.__support.OSUtil.osutil + libc.src.__support.common + libc.src.__support.CPP.optional + libc.src.__support.threads.callonce +) + add_header_library( getrandom HDRS diff --git a/libc/src/__support/OSUtil/linux/auxv.h b/libc/src/__support/OSUtil/linux/auxv.h new file mode 100644 index 0000000000000..17108e2e54794 --- /dev/null +++ b/libc/src/__support/OSUtil/linux/auxv.h @@ -0,0 +1,151 @@ +//===------------- Linux AUXV Header --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H + +#include "hdr/fcntl_macros.h" // For open flags +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/common.h" +#include "src/__support/threads/callonce.h" + +#include // For AT_ macros +#include // For mmap flags +#include // For prctl +#include // For syscall numbers + +namespace LIBC_NAMESPACE_DECL { + +namespace auxv { +struct Entry { + unsigned long type; // Entry type + unsigned long val; // Integer value +}; + +class Vector { + LIBC_INLINE_VAR static constexpr Entry END = {AT_NULL, AT_NULL}; + LIBC_INLINE_VAR static const Entry *entries = &END; + LIBC_INLINE_VAR static CallOnceFlag init_flag = callonce_impl::NOT_CALLED; + LIBC_INLINE_VAR constexpr static size_t FALLBACK_AUXV_ENTRIES = 64; + + LIBC_INLINE static void fallback_initialize_unsync(); + LIBC_INLINE static const Entry *get_entries() { + if (LIBC_LIKELY(entries != &END)) + return entries; + callonce(&init_flag, fallback_initialize_unsync); + return entries; + } + +public: + class Iterator { + const Entry *current; + + public: + LIBC_INLINE explicit Iterator(const Entry *entry) : current(entry) {} + LIBC_INLINE Iterator &operator++() { + ++current; + return *this; + } + LIBC_INLINE const Entry &operator*() const { return *current; } + LIBC_INLINE bool operator!=(const Iterator &other) const { + return current->type != other.current->type; + } + LIBC_INLINE bool operator==(const Iterator &other) const { + return current->type == other.current->type; + } + }; + using iterator = Iterator; + LIBC_INLINE static Iterator begin() { return Iterator(get_entries()); } + LIBC_INLINE static Iterator end() { return Iterator(&END); } + LIBC_INLINE static void initialize_unsafe(const Entry *auxv); +}; + +// Initializes the auxv entries. +// This function is intended to be called once inside crt0. +LIBC_INLINE void Vector::initialize_unsafe(const Entry *auxv) { + init_flag = callonce_impl::FINISH; + entries = auxv; +} + +// When CRT0 does not setup the global array, this function is called. +// As its name suggests, this function is not thread-safe and should be +// backed by a callonce guard. +// This initialize routine will do a mmap to allocate a memory region. +// Since auxv tends to live throughout the program lifetime, we do not +// munmap it. +[[gnu::cold]] +LIBC_INLINE void Vector::fallback_initialize_unsync() { + constexpr size_t AUXV_MMAP_SIZE = FALLBACK_AUXV_ENTRIES * sizeof(Entry); + long mmap_ret = syscall_impl(SYS_mmap, nullptr, AUXV_MMAP_SIZE, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + // We do not proceed if mmap fails. + if (mmap_ret <= 0) + return; + + // Initialize the auxv array with AT_NULL entries. + Entry *vector = reinterpret_cast(mmap_ret); + for (size_t i = 0; i < FALLBACK_AUXV_ENTRIES; ++i) { + vector[i].type = AT_NULL; + vector[i].val = AT_NULL; + } + size_t avaiable_size = AUXV_MMAP_SIZE - sizeof(Entry); + + // Attempt 1: use PRCTL to get the auxv. + // We guarantee that the vector is always padded with AT_NULL entries. + long prctl_ret = syscall_impl(SYS_prctl, PR_GET_AUXV, + reinterpret_cast(vector), + avaiable_size, 0, 0); + if (prctl_ret >= 0) { + entries = vector; + return; + } + + // Attempt 2: read /proc/self/auxv. +#ifdef SYS_openat + int fd = syscall_impl(SYS_openat, AT_FDCWD, "/proc/self/auxv", + O_RDONLY | O_CLOEXEC); +#else + int fd = syscall_impl(SYS_open, "/proc/self/auxv", O_RDONLY | O_CLOEXEC); +#endif + if (fd < 0) { + syscall_impl(SYS_munmap, vector, AUXV_MMAP_SIZE); + return; + } + uint8_t *cursor = reinterpret_cast(vector); + bool has_error = false; + while (avaiable_size != 0) { + long bytes_read = syscall_impl(SYS_read, fd, cursor, avaiable_size); + if (bytes_read <= 0) { + if (bytes_read == -EINTR) + continue; + has_error = bytes_read < 0; + break; + } + avaiable_size -= bytes_read; + cursor += bytes_read; + } + syscall_impl(SYS_close, fd); + if (has_error) { + syscall_impl(SYS_munmap, vector, AUXV_MMAP_SIZE); + return; + } + entries = vector; +} + +LIBC_INLINE cpp::optional get(unsigned long type) { + Vector auxvec; + for (const auto &entry : auxvec) + if (entry.type == type) + return entry.val; + return cpp::nullopt; +} +} // namespace auxv +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h index 5392722045c94..0ccbcf96b995a 100644 --- a/libc/src/__support/threads/callonce.h +++ b/libc/src/__support/threads/callonce.h @@ -23,14 +23,8 @@ #endif namespace LIBC_NAMESPACE_DECL { - -// Common definitions -using CallOnceCallback = void(void); -namespace callonce_impl { -int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback); -} // namespace callonce_impl - -LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) { +template +LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback callback) { if (LIBC_LIKELY(callonce_impl::callonce_fastpath(flag))) return 0; diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt index 14aaad214f43a..39d2c6fef0fed 100644 --- a/libc/src/__support/threads/linux/CMakeLists.txt +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -97,10 +97,8 @@ add_object_library( # value other than 0 is dangerous. We know. ) -add_object_library( +add_header_library( callonce - SRCS - callonce.cpp HDRS ../callonce.h callonce.h diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp deleted file mode 100644 index c6e5f2aba69fc..0000000000000 --- a/libc/src/__support/threads/linux/callonce.cpp +++ /dev/null @@ -1,40 +0,0 @@ -//===-- Linux implementation of the callonce function ---------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "src/__support/threads/callonce.h" -#include "src/__support/macros/config.h" -#include "src/__support/threads/linux/callonce.h" -#include "src/__support/threads/linux/futex_utils.h" - -namespace LIBC_NAMESPACE_DECL { -namespace callonce_impl { -int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *func) { - auto *futex_word = reinterpret_cast(flag); - - FutexWordType not_called = NOT_CALLED; - - // The call_once call can return only after the called function |func| - // returns. So, we use futexes to synchronize calls with the same flag value. - if (futex_word->compare_exchange_strong(not_called, START)) { - func(); - auto status = futex_word->exchange(FINISH); - if (status == WAITING) - futex_word->notify_all(); - return 0; - } - - FutexWordType status = START; - if (futex_word->compare_exchange_strong(status, WAITING) || - status == WAITING) { - futex_word->wait(WAITING); - } - - return 0; -} -} // namespace callonce_impl -} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/threads/linux/callonce.h b/libc/src/__support/threads/linux/callonce.h index b3ea3a5b1c679..70872b6d90156 100644 --- a/libc/src/__support/threads/linux/callonce.h +++ b/libc/src/__support/threads/linux/callonce.h @@ -26,6 +26,31 @@ static constexpr FutexWordType FINISH = 0x33; LIBC_INLINE bool callonce_fastpath(CallOnceFlag *flag) { return flag->load(cpp::MemoryOrder::RELAXED) == FINISH; } + +template +[[gnu::noinline, gnu::cold]] int callonce_slowpath(CallOnceFlag *flag, + CallOnceCallback callback) { + + auto *futex_word = reinterpret_cast(flag); + + FutexWordType not_called = NOT_CALLED; + + // The call_once call can return only after the called function |func| + // returns. So, we use futexes to synchronize calls with the same flag value. + if (futex_word->compare_exchange_strong(not_called, START)) { + callback(); + auto status = futex_word->exchange(FINISH); + if (status == WAITING) + futex_word->notify_all(); + return 0; + } + + FutexWordType status = START; + if (futex_word->compare_exchange_strong(status, WAITING) || status == WAITING) + futex_word->wait(WAITING); + + return 0; +} } // namespace callonce_impl } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/pthread/pthread_once.cpp b/libc/src/pthread/pthread_once.cpp index d78644a8dab26..109f70c24f1cf 100644 --- a/libc/src/pthread/pthread_once.cpp +++ b/libc/src/pthread/pthread_once.cpp @@ -17,8 +17,7 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(int, pthread_once, (pthread_once_t * flag, __pthread_once_func_t func)) { - return callonce(reinterpret_cast(flag), - reinterpret_cast(func)); + return callonce(reinterpret_cast(flag), func); } } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/sys/auxv/linux/CMakeLists.txt b/libc/src/sys/auxv/linux/CMakeLists.txt index 4884184cc6053..39cfb7ba3f7ee 100644 --- a/libc/src/sys/auxv/linux/CMakeLists.txt +++ b/libc/src/sys/auxv/linux/CMakeLists.txt @@ -5,15 +5,6 @@ add_entrypoint_object( HDRS ../getauxval.h DEPENDS - libc.src.sys.prctl.prctl - libc.src.sys.mman.mmap - libc.src.sys.mman.munmap - libc.src.__support.threads.callonce - libc.src.__support.common - libc.src.errno.errno - libc.config.app_h - libc.src.fcntl.open - libc.src.unistd.read - libc.src.unistd.close - libc.include.sys_auxv + libc.src.__support.OSUtil.linux.auxv + libc.src.__support.libc_errno ) diff --git a/libc/src/sys/auxv/linux/getauxval.cpp b/libc/src/sys/auxv/linux/getauxval.cpp index b50c5845bcc2b..52d5a988a9494 100644 --- a/libc/src/sys/auxv/linux/getauxval.cpp +++ b/libc/src/sys/auxv/linux/getauxval.cpp @@ -7,227 +7,14 @@ //===----------------------------------------------------------------------===// #include "src/sys/auxv/getauxval.h" -#include "config/app.h" -#include "hdr/fcntl_macros.h" -#include "src/__support/OSUtil/fcntl.h" -#include "src/__support/common.h" +#include "src/__support/OSUtil/linux/auxv.h" #include "src/__support/libc_errno.h" -#include "src/__support/macros/config.h" -#include - -// for guarded initialization -#include "src/__support/threads/callonce.h" -#include "src/__support/threads/linux/futex_word.h" - -// ----------------------------------------------------------------------------- -// TODO: This file should not include other public libc functions. Calling other -// public libc functions is an antipattern within LLVM-libc. This needs to be -// cleaned up. DO NOT COPY THIS. -// ----------------------------------------------------------------------------- - -// for mallocing the global auxv -#include "src/sys/mman/mmap.h" -#include "src/sys/mman/munmap.h" - -// for reading /proc/self/auxv -#include "src/sys/prctl/prctl.h" -#include "src/unistd/read.h" - -// getauxval will work either with or without __cxa_atexit support. -// In order to detect if __cxa_atexit is supported, we define a weak symbol. -// We prefer __cxa_atexit as it is always defined as a C symbol whileas atexit -// may not be created via objcopy yet. Also, for glibc, atexit is provided via -// libc_nonshared.a rather than libc.so. So, it is may not be made ready for -// overlay builds. -extern "C" [[gnu::weak]] int __cxa_atexit(void (*callback)(void *), - void *payload, void *); namespace LIBC_NAMESPACE_DECL { - -constexpr static size_t MAX_AUXV_ENTRIES = 64; - -// Helper to recover or set errno -class AuxvErrnoGuard { -public: - AuxvErrnoGuard() : saved(libc_errno), failure(false) {} - ~AuxvErrnoGuard() { libc_errno = failure ? ENOENT : saved; } - void mark_failure() { failure = true; } - -private: - int saved; - bool failure; -}; - -// Helper to manage the memory -static AuxEntry *auxv = nullptr; - -class AuxvMMapGuard { -public: - constexpr static size_t AUXV_MMAP_SIZE = sizeof(AuxEntry) * MAX_AUXV_ENTRIES; - - AuxvMMapGuard() - : ptr(LIBC_NAMESPACE::mmap(nullptr, AUXV_MMAP_SIZE, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) {} - ~AuxvMMapGuard() { - if (ptr != MAP_FAILED) - LIBC_NAMESPACE::munmap(ptr, AUXV_MMAP_SIZE); - } - void submit_to_global() { - // atexit may fail, we do not set it to global in that case. - int ret = __cxa_atexit( - [](void *) { - LIBC_NAMESPACE::munmap(auxv, AUXV_MMAP_SIZE); - auxv = nullptr; - }, - nullptr, nullptr); - - if (ret != 0) - return; - - auxv = reinterpret_cast(ptr); - ptr = MAP_FAILED; - } - bool allocated() const { return ptr != MAP_FAILED; } - void *get() const { return ptr; } - -private: - void *ptr; -}; - -class AuxvFdGuard { -public: - AuxvFdGuard() { - auto result = internal::open("/proc/self/auxv", O_RDONLY | O_CLOEXEC); - if (!result.has_value()) - fd = -1; - - fd = result.value(); - } - ~AuxvFdGuard() { - if (fd != -1) - internal::close(fd); - } - bool valid() const { return fd != -1; } - int get() const { return fd; } - -private: - int fd; -}; - -static void initialize_auxv_once(void) { - // If we cannot get atexit, we cannot register the cleanup function. - if (&__cxa_atexit == nullptr) - return; - - AuxvMMapGuard mmap_guard; - if (!mmap_guard.allocated()) - return; - auto *ptr = reinterpret_cast(mmap_guard.get()); - - // We get one less than the max size to make sure the search always - // terminates. MMAP private pages are zeroed out already. - size_t available_size = AuxvMMapGuard::AUXV_MMAP_SIZE - sizeof(AuxEntryType); - // PR_GET_AUXV is only available on Linux kernel 6.1 and above. If this is not - // defined, we direcly fall back to reading /proc/self/auxv. In case the libc - // is compiled and run on separate kernels, we also check the return value of - // prctl. -#ifdef PR_GET_AUXV - int ret = prctl(PR_GET_AUXV, reinterpret_cast(ptr), - available_size, 0, 0); - if (ret >= 0) { - mmap_guard.submit_to_global(); - return; - } -#endif - AuxvFdGuard fd_guard; - if (!fd_guard.valid()) - return; - auto *buf = reinterpret_cast(ptr); - libc_errno = 0; - bool error_detected = false; - // Read until we use up all the available space or we finish reading the file. - while (available_size != 0) { - ssize_t bytes_read = - LIBC_NAMESPACE::read(fd_guard.get(), buf, available_size); - if (bytes_read <= 0) { - if (libc_errno == EINTR) - continue; - // Now, we either have an non-recoverable error or we have reached the end - // of the file. Mark `error_detected` accordingly. - if (bytes_read == -1) - error_detected = true; - break; - } - buf += bytes_read; - available_size -= bytes_read; - } - // If we get out of the loop without an error, the auxv is ready. - if (!error_detected) - mmap_guard.submit_to_global(); -} - -static AuxEntry read_entry(int fd) { - AuxEntry buf; - size_t size = sizeof(AuxEntry); - char *ptr = reinterpret_cast(&buf); - while (size > 0) { - ssize_t ret = LIBC_NAMESPACE::read(fd, ptr, size); - if (ret < 0) { - if (libc_errno == EINTR) - continue; - // Error detected, return AT_NULL - buf.id = AT_NULL; - buf.value = AT_NULL; - break; - } - ptr += ret; - size -= ret; - } - return buf; -} - LLVM_LIBC_FUNCTION(unsigned long, getauxval, (unsigned long id)) { - // Fast path when libc is loaded by its own initialization code. In this case, - // app.auxv_ptr is already set to the auxv passed on the initial stack of the - // process. - AuxvErrnoGuard errno_guard; - - auto search_auxv = [&errno_guard](AuxEntry *auxv, - unsigned long id) -> AuxEntryType { - for (auto *ptr = auxv; ptr->id != AT_NULL; ptr++) - if (ptr->id == id) - return ptr->value; - - errno_guard.mark_failure(); - return AT_NULL; - }; - - // App is a weak symbol that is only defined if libc is linked to its own - // initialization routine. We need to check if it is null. - if (&app != nullptr) - return search_auxv(app.auxv_ptr, id); - - static FutexWordType once_flag; - LIBC_NAMESPACE::callonce(reinterpret_cast(&once_flag), - initialize_auxv_once); - if (auxv != nullptr) - return search_auxv(auxv, id); - - // Fallback to use read without mmap - AuxvFdGuard fd_guard; - if (fd_guard.valid()) { - while (true) { - AuxEntry buf = read_entry(fd_guard.get()); - if (buf.id == AT_NULL) - break; - if (buf.id == id) - return buf.value; - } - } - - // cannot find the entry after all methods, mark failure and return 0 - errno_guard.mark_failure(); - return AT_NULL; + if (cpp::optional val = auxv::get(id)) + return *val; + libc_errno = ENOENT; + return 0; } } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/threads/call_once.cpp b/libc/src/threads/call_once.cpp index 8466cd622afeb..b60a20d1cdd15 100644 --- a/libc/src/threads/call_once.cpp +++ b/libc/src/threads/call_once.cpp @@ -17,8 +17,7 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void, call_once, (once_flag * flag, __call_once_func_t func)) { - callonce(reinterpret_cast(flag), - reinterpret_cast(func)); + callonce(reinterpret_cast(flag), func); } } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/startup/linux/CMakeLists.txt b/libc/startup/linux/CMakeLists.txt index 7af1819c57c17..df2c4b9ec508c 100644 --- a/libc/startup/linux/CMakeLists.txt +++ b/libc/startup/linux/CMakeLists.txt @@ -105,6 +105,7 @@ add_object_library( libc.src.stdlib.exit libc.src.stdlib.atexit libc.src.unistd.environ + libc.src.__support.OSUtil.linux.auxv COMPILE_OPTIONS -ffreestanding # To avoid compiler warnings about calling the main function. -fno-builtin # avoid emit unexpected calls diff --git a/libc/startup/linux/do_start.cpp b/libc/startup/linux/do_start.cpp index 94c4ec7092861..a67bf188b1ae3 100644 --- a/libc/startup/linux/do_start.cpp +++ b/libc/startup/linux/do_start.cpp @@ -9,6 +9,7 @@ #include "config/linux/app.h" #include "hdr/stdint_proxy.h" #include "include/llvm-libc-macros/link-macros.h" +#include "src/__support/OSUtil/linux/auxv.h" #include "src/__support/OSUtil/syscall.h" #include "src/__support/macros/config.h" #include "src/__support/threads/thread.h" @@ -88,17 +89,19 @@ void teardown_main_tls() { cleanup_tls(tls.addr, tls.size); } // denoted by an AT_NULL entry. ElfW(Phdr) *program_hdr_table = nullptr; uintptr_t program_hdr_count = 0; - app.auxv_ptr = reinterpret_cast(env_end_marker + 1); - for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) { - switch (aux_entry->id) { + auxv::Vector::initialize_unsafe( + reinterpret_cast(env_end_marker + 1)); + auxv::Vector auxvec; + for (const auto &aux_entry : auxvec) { + switch (aux_entry.type) { case AT_PHDR: - program_hdr_table = reinterpret_cast(aux_entry->value); + program_hdr_table = reinterpret_cast(aux_entry.val); break; case AT_PHNUM: - program_hdr_count = aux_entry->value; + program_hdr_count = aux_entry.val; break; case AT_PAGESZ: - app.page_size = aux_entry->value; + app.page_size = aux_entry.val; break; default: break; // TODO: Read other useful entries from the aux vector.