[libc] Refactor AUXV handling with new auxv.h header library

libc/config/linux/app.h

@@ -35,17 +35,6 @@ struct TLSImage {
   uintptr_t align;
 };
 
-// Linux manpage on `proc(5)` says that the aux vector is an array of
-// unsigned long pairs.
-// (see: https://man7.org/linux/man-pages/man5/proc.5.html)
-using AuxEntryType = unsigned long;
-// Using the naming convention from `proc(5)`.
-// TODO: Would be nice to use the aux entry structure from elf.h when available.
-struct AuxEntry {
-  AuxEntryType id;
-  AuxEntryType value;
-};
-
 struct Args {
   uintptr_t argc;
 
@@ -70,9 +59,6 @@ struct AppProperties {
 
   // Environment data.
   uintptr_t *env_ptr;
-
-  // Auxiliary vector data.
-  AuxEntry *auxv_ptr;
 };
 
 [[gnu::weak]] extern AppProperties app;

libc/src/__support/OSUtil/linux/CMakeLists.txt

@@ -24,6 +24,18 @@ add_object_library(
     libc.include.sys_syscall
 )
 
+add_header_library(
+  auxv
+  HDRS
+    auxv.h
+  DEPENDS
+    libc.hdr.fcntl_macros
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.common
+    libc.src.__support.CPP.optional
+    libc.src.__support.threads.callonce
+)
+
 add_header_library(
   getrandom
   HDRS

libc/src/__support/OSUtil/linux/auxv.h

@@ -0,0 +1,151 @@
+//===------------- Linux AUXV Header --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H
+
+#include "hdr/fcntl_macros.h" // For open flags
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/common.h"
+#include "src/__support/threads/callonce.h"
+
+#include <linux/auxvec.h> // For AT_ macros
+#include <linux/mman.h>   // For mmap flags
+#include <linux/prctl.h>  // For prctl
+#include <sys/syscall.h>  // For syscall numbers
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace auxv {
+struct Entry {
+  unsigned long type; // Entry type
+  unsigned long val;  // Integer value
+};
+
+class Vector {
+  LIBC_INLINE_VAR static constexpr Entry END = {AT_NULL, AT_NULL};
+  LIBC_INLINE_VAR static const Entry *entries = &END;
+  LIBC_INLINE_VAR static CallOnceFlag init_flag = callonce_impl::NOT_CALLED;
+  LIBC_INLINE_VAR constexpr static size_t FALLBACK_AUXV_ENTRIES = 64;
+
+  LIBC_INLINE static void fallback_initialize_unsync();
+  LIBC_INLINE static const Entry *get_entries() {
+    if (LIBC_LIKELY(entries != &END))
+      return entries;
+    callonce(&init_flag, fallback_initialize_unsync);
+    return entries;
+  }
+
+public:
+  class Iterator {
+    const Entry *current;
+
+  public:
+    LIBC_INLINE explicit Iterator(const Entry *entry) : current(entry) {}
+    LIBC_INLINE Iterator &operator++() {
+      ++current;
+      return *this;
+    }
+    LIBC_INLINE const Entry &operator*() const { return *current; }
+    LIBC_INLINE bool operator!=(const Iterator &other) const {
+      return current->type != other.current->type;
+    }
+    LIBC_INLINE bool operator==(const Iterator &other) const {
+      return current->type == other.current->type;
+    }
+  };
+  using iterator = Iterator;
+  LIBC_INLINE static Iterator begin() { return Iterator(get_entries()); }
+  LIBC_INLINE static Iterator end() { return Iterator(&END); }
+  LIBC_INLINE static void initialize_unsafe(const Entry *auxv);
+};
+
+// Initializes the auxv entries.
+// This function is intended to be called once inside crt0.
+LIBC_INLINE void Vector::initialize_unsafe(const Entry *auxv) {
+  init_flag = callonce_impl::FINISH;
+  entries = auxv;
+}
+
+// When CRT0 does not setup the global array, this function is called.
+// As its name suggests, this function is not thread-safe and should be
+// backed by a callonce guard.
+// This initialize routine will do a mmap to allocate a memory region.
+// Since auxv tends to live throughout the program lifetime, we do not
+// munmap it.
+[[gnu::cold]]
+LIBC_INLINE void Vector::fallback_initialize_unsync() {
+  constexpr size_t AUXV_MMAP_SIZE = FALLBACK_AUXV_ENTRIES * sizeof(Entry);
+  long mmap_ret = syscall_impl<long>(SYS_mmap, nullptr, AUXV_MMAP_SIZE,
+                                     PROT_READ | PROT_WRITE,
+                                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  // We do not proceed if mmap fails.
+  if (mmap_ret <= 0)
+    return;
+
+  // Initialize the auxv array with AT_NULL entries.
+  Entry *vector = reinterpret_cast<Entry *>(mmap_ret);
+  for (size_t i = 0; i < FALLBACK_AUXV_ENTRIES; ++i) {
+    vector[i].type = AT_NULL;
+    vector[i].val = AT_NULL;
+  }
+  size_t avaiable_size = AUXV_MMAP_SIZE - sizeof(Entry);
+
+  // Attempt 1: use PRCTL to get the auxv.
+  // We guarantee that the vector is always padded with AT_NULL entries.
+  long prctl_ret = syscall_impl<long>(SYS_prctl, PR_GET_AUXV,
+                                      reinterpret_cast<unsigned long>(vector),
+                                      avaiable_size, 0, 0);
+  if (prctl_ret >= 0) {
+    entries = vector;
+    return;
+  }
+
+  // Attempt 2: read /proc/self/auxv.
+#ifdef SYS_openat
+  int fd = syscall_impl<int>(SYS_openat, AT_FDCWD, "/proc/self/auxv",
+                             O_RDONLY | O_CLOEXEC);
+#else
+  int fd = syscall_impl<int>(SYS_open, "/proc/self/auxv", O_RDONLY | O_CLOEXEC);
+#endif
+  if (fd < 0) {
+    syscall_impl<long>(SYS_munmap, vector, AUXV_MMAP_SIZE);
+    return;
+  }
+  uint8_t *cursor = reinterpret_cast<uint8_t *>(vector);
+  bool has_error = false;
+  while (avaiable_size != 0) {
+    long bytes_read = syscall_impl<long>(SYS_read, fd, cursor, avaiable_size);
+    if (bytes_read <= 0) {
+      if (bytes_read == -EINTR)
+        continue;
+      has_error = bytes_read < 0;
+      break;
+    }
+    avaiable_size -= bytes_read;
+    cursor += bytes_read;
+  }
+  syscall_impl<long>(SYS_close, fd);
+  if (has_error) {
+    syscall_impl<long>(SYS_munmap, vector, AUXV_MMAP_SIZE);
+    return;
+  }
+  entries = vector;
+}
+
+LIBC_INLINE cpp::optional<unsigned long> get(unsigned long type) {
+  Vector auxvec;
+  for (const auto &entry : auxvec)
+    if (entry.type == type)
+      return entry.val;
+  return cpp::nullopt;
+}
+} // namespace auxv
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H

libc/src/__support/threads/callonce.h

@@ -23,14 +23,8 @@
 #endif
 
 namespace LIBC_NAMESPACE_DECL {
-
-// Common definitions
-using CallOnceCallback = void(void);
-namespace callonce_impl {
-int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback);
-} // namespace callonce_impl
-
-LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) {
+template <class CallOnceCallback>
+LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback callback) {
   if (LIBC_LIKELY(callonce_impl::callonce_fastpath(flag)))
     return 0;
 

libc/src/__support/threads/linux/CMakeLists.txt

@@ -97,10 +97,8 @@ add_object_library(
                             # value other than 0 is dangerous. We know.
 )
 
-add_object_library(
+add_header_library(
   callonce
-  SRCS
-    callonce.cpp
   HDRS
     ../callonce.h
     callonce.h

libc/src/__support/threads/linux/callonce.h

@@ -26,6 +26,31 @@ static constexpr FutexWordType FINISH = 0x33;
 LIBC_INLINE bool callonce_fastpath(CallOnceFlag *flag) {
   return flag->load(cpp::MemoryOrder::RELAXED) == FINISH;
 }
+
+template <class CallOnceCallback>
+[[gnu::noinline, gnu::cold]] int callonce_slowpath(CallOnceFlag *flag,
+                                                   CallOnceCallback callback) {
+
+  auto *futex_word = reinterpret_cast<Futex *>(flag);
+
+  FutexWordType not_called = NOT_CALLED;
+
+  // The call_once call can return only after the called function |func|
+  // returns. So, we use futexes to synchronize calls with the same flag value.
+  if (futex_word->compare_exchange_strong(not_called, START)) {
+    callback();
+    auto status = futex_word->exchange(FINISH);
+    if (status == WAITING)
+      futex_word->notify_all();
+    return 0;
+  }
+
+  FutexWordType status = START;
+  if (futex_word->compare_exchange_strong(status, WAITING) || status == WAITING)
+    futex_word->wait(WAITING);
+
+  return 0;
+}
 } // namespace callonce_impl
 
 } // namespace LIBC_NAMESPACE_DECL

libc/src/pthread/pthread_once.cpp

@@ -17,8 +17,7 @@ namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(int, pthread_once,
                    (pthread_once_t * flag, __pthread_once_func_t func)) {
-  return callonce(reinterpret_cast<CallOnceFlag *>(flag),
-                  reinterpret_cast<CallOnceCallback *>(func));
+  return callonce(reinterpret_cast<CallOnceFlag *>(flag), func);
 }
 
 } // namespace LIBC_NAMESPACE_DECL

libc/src/sys/auxv/linux/CMakeLists.txt

@@ -5,15 +5,6 @@ add_entrypoint_object(
   HDRS
     ../getauxval.h
   DEPENDS
-    libc.src.sys.prctl.prctl
-    libc.src.sys.mman.mmap
-    libc.src.sys.mman.munmap
-    libc.src.__support.threads.callonce
-    libc.src.__support.common
-    libc.src.errno.errno
-    libc.config.app_h
-    libc.src.fcntl.open
-    libc.src.unistd.read
-    libc.src.unistd.close
-    libc.include.sys_auxv
+    libc.src.__support.OSUtil.linux.auxv
+    libc.src.__support.libc_errno
 )