diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt index a2fad9b473ed7e..e0b65b7c2eb02d 100644 --- a/libc/hdr/CMakeLists.txt +++ b/libc/hdr/CMakeLists.txt @@ -143,4 +143,22 @@ add_proxy_header_library( libc.include.llvm-libc-macros.limits_macros ) +add_proxy_header_library( + link_macros + HDRS + link_macros.h + FULL_BUILD_DEPENDS + libc.include.llvm-libc-macros.link_macros + libc.include.link +) + +add_proxy_header_library( + sys_auxv_macros + HDRS + sys_auxv_macros.h + FULL_BUILD_DEPENDS + libc.include.llvm-libc-macros.sys_auxv_macros + libc.include.sys_auxv +) + add_subdirectory(types) diff --git a/libc/hdr/link_macros.h b/libc/hdr/link_macros.h new file mode 100644 index 00000000000000..8a78a864e6ce4c --- /dev/null +++ b/libc/hdr/link_macros.h @@ -0,0 +1,22 @@ +//===-- Definition of macros from link.h ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_LINK_MACROS_H +#define LLVM_LIBC_HDR_LINK_MACROS_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-macros/link-macros.h" + +#else // Overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_LINK_MACROS_H diff --git a/libc/hdr/sys_auxv_macros.h b/libc/hdr/sys_auxv_macros.h new file mode 100644 index 00000000000000..c04011baedb860 --- /dev/null +++ b/libc/hdr/sys_auxv_macros.h @@ -0,0 +1,22 @@ +//===-- Definition of macros from sys/auxv.h ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_SYS_AUXV_MACROS_H +#define LLVM_LIBC_HDR_SYS_AUXV_MACROS_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-macros/sys-auxv-macros.h" + +#else // Overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_SYS_AUXV_MACROS_H diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt index 089cad454d534d..6c7014940407d8 100644 --- a/libc/src/__support/OSUtil/linux/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt @@ -23,3 +23,33 @@ add_object_library( libc.hdr.types.struct_f_owner_ex libc.hdr.types.off_t ) + +add_header_library( + vdso_sym + HDRS + vdso_sym.h + DEPENDS + libc.src.__support.common +) + +add_object_library( + vdso + HDRS + vdso.h + SRCS + vdso.cpp + DEPENDS + .${LIBC_TARGET_ARCHITECTURE}.vdso + libc.src.__support.CPP.array + libc.src.__support.CPP.optional + libc.src.__support.CPP.string_view + libc.src.__support.threads.callonce + libc.src.__support.threads.linux.futex_word_type + libc.hdr.types.struct_timeval + libc.hdr.types.struct_timespec + libc.hdr.types.clockid_t + libc.hdr.types.time_t + libc.hdr.link_macros + libc.src.errno.errno + libc.src.sys.auxv.getauxval +) diff --git a/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt b/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt index eea9badc46cae6..d9451a1af1df35 100644 --- a/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/aarch64/CMakeLists.txt @@ -5,3 +5,13 @@ add_header_library( DEPENDS libc.src.__support.common ) + +add_header_library( + vdso + HDRS + vdso.h + DEPENDS + libc.src.__support.common + libc.src.__support.CPP.string_view + libc.src.__support.OSUtil.linux.vdso_sym +) diff --git a/libc/src/__support/OSUtil/linux/aarch64/vdso.h b/libc/src/__support/OSUtil/linux/aarch64/vdso.h new file mode 100644 index 00000000000000..3c4c6205071da2 --- /dev/null +++ b/libc/src/__support/OSUtil/linux/aarch64/vdso.h @@ -0,0 +1,37 @@ +//===---------- aarch64 vdso configuration ------------------------* C++ *-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AARCH64_VDSO_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AARCH64_VDSO_H +#include "src/__support/CPP/string_view.h" +#include "src/__support/OSUtil/linux/vdso_sym.h" +namespace LIBC_NAMESPACE_DECL { +namespace vdso { +// translate VDSOSym to symbol names +// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm64/kernel/vdso/vdso.lds.S +LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) { + switch (sym) { + case VDSOSym::RTSigReturn: + return "__kernel_rt_sigreturn"; + case VDSOSym::GetTimeOfDay: + return "__kernel_gettimeofday"; + case VDSOSym::ClockGetTime: + return "__kernel_clock_gettime"; + case VDSOSym::ClockGetRes: + return "__kernel_clock_getres"; + default: + return ""; + } +} + +// symbol versions +LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) { + return "LINUX_2.6.39"; +} +} // namespace vdso +} // namespace LIBC_NAMESPACE_DECL +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AARCH64_VDSO_H diff --git a/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt b/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt index 733366f6d4a2e3..d991f7e0914796 100644 --- a/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/arm/CMakeLists.txt @@ -5,3 +5,13 @@ add_header_library( DEPENDS libc.src.__support.common ) + +add_header_library( + vdso + HDRS + vdso.h + DEPENDS + libc.src.__support.common + libc.src.__support.CPP.string_view + libc.src.__support.OSUtil.linux.vdso_sym +) diff --git a/libc/src/__support/OSUtil/linux/arm/vdso.h b/libc/src/__support/OSUtil/linux/arm/vdso.h new file mode 100644 index 00000000000000..3de5860359c155 --- /dev/null +++ b/libc/src/__support/OSUtil/linux/arm/vdso.h @@ -0,0 +1,37 @@ +//===---------- arm vdso configuration ----------------------------* C++ *-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_ARM_VDSO_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_ARM_VDSO_H +#include "src/__support/CPP/string_view.h" +#include "src/__support/OSUtil/linux/vdso_sym.h" +namespace LIBC_NAMESPACE_DECL { +namespace vdso { +// translate VDSOSym to symbol names +// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm/vdso/vdso.lds.S +LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) { + switch (sym) { + case VDSOSym::ClockGetTime: + return "__vdso_clock_gettime"; + case VDSOSym::GetTimeOfDay: + return "__vdso_gettimeofday"; + case VDSOSym::ClockGetRes: + return "__vdso_clock_getres"; + case VDSOSym::ClockGetTime64: + return "__vdso_clock_gettime64"; + default: + return ""; + } +} + +// symbol versions +LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) { + return "LINUX_2.6"; +} +} // namespace vdso +} // namespace LIBC_NAMESPACE_DECL +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_ARM_VDSO_H diff --git a/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt b/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt index e271204f519820..eb93dd4af35ce7 100644 --- a/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/riscv/CMakeLists.txt @@ -5,3 +5,13 @@ add_header_library( DEPENDS libc.src.__support.common ) + +add_header_library( + vdso + HDRS + vdso.h + DEPENDS + libc.src.__support.common + libc.src.__support.CPP.string_view + libc.src.__support.OSUtil.linux.vdso_sym +) diff --git a/libc/src/__support/OSUtil/linux/riscv/vdso.h b/libc/src/__support/OSUtil/linux/riscv/vdso.h new file mode 100644 index 00000000000000..24ddb25ea980a5 --- /dev/null +++ b/libc/src/__support/OSUtil/linux/riscv/vdso.h @@ -0,0 +1,43 @@ +//===---------- RISC-V vdso configuration -------------------------* C++ *-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_RISCV_VDSO_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_RISCV_VDSO_H +#include "src/__support/CPP/string_view.h" +#include "src/__support/OSUtil/linux/vdso_sym.h" +namespace LIBC_NAMESPACE_DECL { +namespace vdso { +// translate VDSOSym to symbol names +// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/kernel/vdso/vdso.lds.S +LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) { + switch (sym) { + case VDSOSym::RTSigReturn: + return "__vdso_rt_sigreturn"; + case VDSOSym::GetTimeOfDay: + return "__vdso_gettimeofday"; + case VDSOSym::ClockGetTime: + return "__vdso_clock_gettime"; + case VDSOSym::ClockGetRes: + return "__vdso_clock_getres"; + case VDSOSym::GetCpu: + return "__vdso_getcpu"; + case VDSOSym::FlushICache: + return "__vdso_flush_icache"; + case VDSOSym::RiscvHwProbe: + return "__vdso_riscv_hwprobe"; + default: + return ""; + } +} + +// symbol versions +LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) { + return "LINUX_4.15"; +} +} // namespace vdso +} // namespace LIBC_NAMESPACE_DECL +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_RISCV_VDSO_H diff --git a/libc/src/__support/OSUtil/linux/vdso.cpp b/libc/src/__support/OSUtil/linux/vdso.cpp new file mode 100644 index 00000000000000..cb43764badad1f --- /dev/null +++ b/libc/src/__support/OSUtil/linux/vdso.cpp @@ -0,0 +1,237 @@ +//===------------- Linux VDSO Implementation --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "src/__support/OSUtil/linux/vdso.h" +#include "hdr/link_macros.h" +#include "hdr/sys_auxv_macros.h" +#include "src/__support/CPP/array.h" +#include "src/__support/CPP/optional.h" +#include "src/__support/CPP/string_view.h" +#include "src/__support/threads/callonce.h" +#include "src/__support/threads/linux/futex_word.h" +#include "src/errno/libc_errno.h" +#include "src/sys/auxv/getauxval.h" +#include + +// TODO: This is a temporary workaround to avoid including elf.h +// Include our own headers for ElfW and friends once we have them. +namespace LIBC_NAMESPACE_DECL { + +namespace vdso { + +Symbol::VDSOArray Symbol::global_cache{}; +CallOnceFlag Symbol::once_flag = callonce_impl::NOT_CALLED; + +namespace { +// See https://refspecs.linuxfoundation.org/LSB_1.3.0/gLSB/gLSB/symverdefs.html +struct Verdaux { + ElfW(Word) vda_name; /* Version or dependency names */ + ElfW(Word) vda_next; /* Offset in bytes to next verdaux + entry */ +}; +struct Verdef { + ElfW(Half) vd_version; /* Version revision */ + ElfW(Half) vd_flags; /* Version information */ + ElfW(Half) vd_ndx; /* Version Index */ + ElfW(Half) vd_cnt; /* Number of associated aux entries */ + ElfW(Word) vd_hash; /* Version name hash value */ + ElfW(Word) vd_aux; /* Offset in bytes to verdaux array */ + ElfW(Word) vd_next; /* Offset in bytes to next verdef entry */ + Verdef *next() const { + if (vd_next == 0) + return nullptr; + return reinterpret_cast(reinterpret_cast(this) + + vd_next); + } + Verdaux *aux() const { + return reinterpret_cast(reinterpret_cast(this) + + vd_aux); + } +}; + +// version search procedure specified by +// https://refspecs.linuxfoundation.org/LSB_1.3.0/gLSB/gLSB/symversion.html#SYMVERTBL +cpp::string_view find_version(Verdef *verdef, ElfW(Half) * versym, + const char *strtab, size_t idx) { + constexpr ElfW(Half) VER_FLG_BASE = 0x1; + if (!versym) + return ""; + ElfW(Half) identifier = versym[idx] & 0x7FFF; + // iterate through all version definitions + for (Verdef *def = verdef; def != nullptr; def = def->next()) { + // skip if this is a file-level version + if (def->vd_flags & VER_FLG_BASE) + continue; + // check if the version identifier matches. Highest bit is used to determine + // whether the symbol is local. Only lower 15 bits are used for version + // identifier. + if ((def->vd_ndx & 0x7FFF) == identifier) { + Verdaux *aux = def->aux(); + return strtab + aux->vda_name; + } + } + return ""; +} + +size_t shdr_get_symbol_count(ElfW(Shdr) * vdso_shdr, size_t e_shnum) { + if (!vdso_shdr) + return 0; + // iterate all sections until we locate the dynamic symbol section + for (size_t i = 0; i < e_shnum; ++i) { + // dynamic symbol section is a table section + // therefore, the number of entries can be computed as the ratio + // of the section size to the size of a single entry + if (vdso_shdr[i].sh_type == SHT_DYNSYM) + return vdso_shdr[i].sh_size / vdso_shdr[i].sh_entsize; + } + return 0; +} + +struct VDSOSymbolTable { + const char *strtab; + ElfW(Sym) * symtab; + // The following can be nullptr if the vDSO does not have versioning + ElfW(Half) * versym; + Verdef *verdef; + + void populate_symbol_cache(Symbol::VDSOArray &symbol_table, + size_t symbol_count, ElfW(Addr) vdso_addr) { + for (size_t i = 0, e = symbol_table.size(); i < e; ++i) { + Symbol sym = i; + cpp::string_view name = sym.name(); + cpp::string_view version = sym.version(); + if (name.empty()) + continue; + + for (size_t j = 0; j < symbol_count; ++j) { + if (name == strtab + symtab[j].st_name) { + // we find a symbol with desired name + // now we need to check if it has the right version + if (versym && verdef && + version != find_version(verdef, versym, strtab, j)) + continue; + + // put the symbol address into the symbol table + symbol_table[i] = + reinterpret_cast(vdso_addr + symtab[j].st_value); + } + } + } + } +}; + +struct PhdrInfo { + ElfW(Addr) vdso_addr; + ElfW(Dyn) * vdso_dyn; + static cpp::optional from(ElfW(Phdr) * vdso_phdr, size_t e_phnum, + uintptr_t vdso_ehdr_addr) { + constexpr ElfW(Addr) INVALID_ADDR = static_cast(-1); + ElfW(Addr) vdso_addr = INVALID_ADDR; + ElfW(Dyn) *vdso_dyn = nullptr; + if (!vdso_phdr) + return cpp::nullopt; + // iterate through all the program headers until we get the desired pieces + for (size_t i = 0; i < e_phnum; ++i) { + if (vdso_phdr[i].p_type == PT_DYNAMIC) + vdso_dyn = reinterpret_cast(vdso_ehdr_addr + + vdso_phdr[i].p_offset); + + if (vdso_phdr[i].p_type == PT_LOAD) + vdso_addr = + vdso_ehdr_addr + vdso_phdr[i].p_offset - vdso_phdr[i].p_vaddr; + + if (vdso_addr && vdso_dyn) + return PhdrInfo{vdso_addr, vdso_dyn}; + } + + return cpp::nullopt; + } + + cpp::optional populate_symbol_table() { + const char *strtab = nullptr; + ElfW(Sym) *symtab = nullptr; + ElfW(Half) *versym = nullptr; + Verdef *verdef = nullptr; + for (ElfW(Dyn) *d = vdso_dyn; d->d_tag != DT_NULL; ++d) { + switch (d->d_tag) { + case DT_STRTAB: + strtab = reinterpret_cast(vdso_addr + d->d_un.d_ptr); + break; + case DT_SYMTAB: + symtab = reinterpret_cast(vdso_addr + d->d_un.d_ptr); + break; + case DT_VERSYM: + versym = reinterpret_cast(vdso_addr + d->d_un.d_ptr); + break; + case DT_VERDEF: + verdef = reinterpret_cast(vdso_addr + d->d_un.d_ptr); + break; + } + if (strtab && symtab && versym && verdef) + break; + } + if (strtab == nullptr || symtab == nullptr) + return cpp::nullopt; + + return VDSOSymbolTable{strtab, symtab, versym, verdef}; + } +}; +} // namespace + +void Symbol::initialize_vdso_global_cache() { + // first clear the symbol table + for (auto &i : global_cache) + i = nullptr; + + // get the address of the VDSO, protect errno since getauxval may change + // it + int errno_backup = libc_errno; + uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR); + // Get the memory address of the vDSO ELF header. + auto vdso_ehdr = reinterpret_cast(vdso_ehdr_addr); + // leave the table unpopulated if we don't have vDSO + if (vdso_ehdr == nullptr) { + libc_errno = errno_backup; + return; + } + + // locate the section header inside the elf using the section header + // offset + auto vdso_shdr = + reinterpret_cast(vdso_ehdr_addr + vdso_ehdr->e_shoff); + size_t symbol_count = shdr_get_symbol_count(vdso_shdr, vdso_ehdr->e_shnum); + + // early return if no symbol is found + if (symbol_count == 0) + return; + + // We need to find both the loadable segment and the dynamic linking of + // the vDSO. compute vdso_phdr as the program header using the program + // header offset + ElfW(Phdr) *vdso_phdr = + reinterpret_cast(vdso_ehdr_addr + vdso_ehdr->e_phoff); + cpp::optional phdr_info = + PhdrInfo::from(vdso_phdr, vdso_ehdr->e_phnum, vdso_ehdr_addr); + // early return if either the dynamic linking or the loadable segment is + // not found + if (!phdr_info.has_value()) + return; + + // now, locate several more tables inside the dynmaic linking section + cpp::optional vdso_symbol_table = + phdr_info->populate_symbol_table(); + + // early return if we can't find any required fields of the symbol table + if (!vdso_symbol_table.has_value()) + return; + + // finally, populate the global symbol table cache + vdso_symbol_table->populate_symbol_cache(global_cache, symbol_count, + phdr_info->vdso_addr); +} +} // namespace vdso +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/OSUtil/linux/vdso.h b/libc/src/__support/OSUtil/linux/vdso.h new file mode 100644 index 00000000000000..a5108b3a1fb5d3 --- /dev/null +++ b/libc/src/__support/OSUtil/linux/vdso.h @@ -0,0 +1,81 @@ +//===------------- Linux VDSO Header ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H +#include "src/__support/CPP/array.h" +#include "src/__support/common.h" +#include "src/__support/macros/attributes.h" +#include "src/__support/macros/properties/architectures.h" +#include "src/__support/threads/callonce.h" + +#if defined(LIBC_TARGET_ARCH_IS_X86) +#include "x86_64/vdso.h" +#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) +#include "aarch64/vdso.h" +#elif defined(LIBC_TARGET_ARCH_IS_ARM) +#include "arm/vdso.h" +#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) +#include "riscv/vdso.h" +#else +#error "unknown arch" +#endif + +namespace LIBC_NAMESPACE_DECL { +namespace vdso { + +class Symbol { + VDSOSym sym; + +public: + LIBC_INLINE_VAR static constexpr size_t COUNT = + static_cast(VDSOSym::VDSOSymCount); + LIBC_INLINE constexpr explicit Symbol(VDSOSym sym) : sym(sym) {} + LIBC_INLINE constexpr Symbol(size_t idx) : sym(static_cast(idx)) {} + LIBC_INLINE constexpr cpp::string_view name() const { + return symbol_name(sym); + } + LIBC_INLINE constexpr cpp::string_view version() const { + return symbol_version(sym); + } + LIBC_INLINE constexpr operator size_t() const { + return static_cast(sym); + } + LIBC_INLINE constexpr bool is_valid() const { + return *this < Symbol::global_cache.size(); + } + using VDSOArray = cpp::array; + +private: + static CallOnceFlag once_flag; + static VDSOArray global_cache; + static void initialize_vdso_global_cache(); + + LIBC_INLINE void *get() const { + if (name().empty() || !is_valid()) + return nullptr; + + callonce(&once_flag, Symbol::initialize_vdso_global_cache); + return (global_cache[*this]); + } + template friend struct TypedSymbol; +}; + +template struct TypedSymbol { + LIBC_INLINE constexpr operator VDSOSymType() const { + return cpp::bit_cast>(Symbol{sym}.get()); + } + template + LIBC_INLINE auto operator()(Args &&...args) const { + return this->operator VDSOSymType()(cpp::forward(args)...); + } +}; + +} // namespace vdso + +} // namespace LIBC_NAMESPACE_DECL +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H diff --git a/libc/src/__support/OSUtil/linux/vdso_sym.h b/libc/src/__support/OSUtil/linux/vdso_sym.h new file mode 100644 index 00000000000000..eb5f204a82f304 --- /dev/null +++ b/libc/src/__support/OSUtil/linux/vdso_sym.h @@ -0,0 +1,70 @@ +//===------------- Linux VDSO Symbols ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "hdr/types/clock_t.h" +#include "hdr/types/clockid_t.h" +#include "hdr/types/struct_timespec.h" +#include "hdr/types/struct_timeval.h" +#include "hdr/types/time_t.h" +#include "src/__support/common.h" +#include // For size_t. + +// NOLINTBEGIN(llvmlibc-implementation-in-namespace) +// TODO: some of the following can be defined via proxy headers. +struct __kernel_timespec; +struct timezone; +struct riscv_hwprobe; +struct getcpu_cache; +struct cpu_set_t; +// NOLINTEND(llvmlibc-implementation-in-namespace) + +namespace LIBC_NAMESPACE_DECL { +namespace vdso { + +enum class VDSOSym { + ClockGetTime, + ClockGetTime64, + GetTimeOfDay, + GetCpu, + Time, + ClockGetRes, + RTSigReturn, + FlushICache, + RiscvHwProbe, + VDSOSymCount +}; + +template LIBC_INLINE constexpr auto dispatcher() { + if constexpr (sym == VDSOSym::ClockGetTime) + return static_cast(nullptr); + else if constexpr (sym == VDSOSym::ClockGetTime64) + return static_cast(nullptr); + else if constexpr (sym == VDSOSym::GetTimeOfDay) + return static_cast( + nullptr); + else if constexpr (sym == VDSOSym::GetCpu) + return static_cast( + nullptr); + else if constexpr (sym == VDSOSym::Time) + return static_cast(nullptr); + else if constexpr (sym == VDSOSym::ClockGetRes) + return static_cast(nullptr); + else if constexpr (sym == VDSOSym::RTSigReturn) + return static_cast(nullptr); + else if constexpr (sym == VDSOSym::FlushICache) + return static_cast(nullptr); + else if constexpr (sym == VDSOSym::RiscvHwProbe) + return static_cast(nullptr); + else + return static_cast(nullptr); +} + +template using VDSOSymType = decltype(dispatcher()); + +} // namespace vdso +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt b/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt index a7f2d74e6353e0..1324491f37b76b 100644 --- a/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/x86_64/CMakeLists.txt @@ -5,3 +5,13 @@ add_header_library( DEPENDS libc.src.__support.common ) + +add_header_library( + vdso + HDRS + vdso.h + DEPENDS + libc.src.__support.common + libc.src.__support.CPP.string_view + libc.src.__support.OSUtil.linux.vdso_sym +) diff --git a/libc/src/__support/OSUtil/linux/x86_64/vdso.h b/libc/src/__support/OSUtil/linux/x86_64/vdso.h new file mode 100644 index 00000000000000..abe7c33e07cfab --- /dev/null +++ b/libc/src/__support/OSUtil/linux/x86_64/vdso.h @@ -0,0 +1,43 @@ +//===---------- x86/x86_64 vdso configuration ---------------------* C++ *-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_X86_64_VDSO_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_X86_64_VDSO_H +#include "src/__support/CPP/string_view.h" +#include "src/__support/OSUtil/linux/vdso_sym.h" +namespace LIBC_NAMESPACE_DECL { +namespace vdso { +// translate VDSOSym to symbol names +// On x86, there are symbols defined without the __vdso_ prefix, however, +// it is suggested that one should use the __vdso_ prefix. +// Additionally, there is also an __vdso_sgx_enter_enclave, it is for the SGX +// support, we do not include it here for now. +// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/entry/vdso/vdso.lds.S +LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) { + switch (sym) { + case VDSOSym::ClockGetTime: + return "__vdso_clock_gettime"; + case VDSOSym::GetTimeOfDay: + return "__vdso_gettimeofday"; + case VDSOSym::GetCpu: + return "__vdso_getcpu"; + case VDSOSym::Time: + return "__vdso_time"; + case VDSOSym::ClockGetRes: + return "__vdso_clock_getres"; + default: + return ""; + } +} + +// symbol versions +LIBC_INLINE constexpr cpp::string_view symbol_version(VDSOSym) { + return "LINUX_2.6"; +} +} // namespace vdso +} // namespace LIBC_NAMESPACE_DECL +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_X86_64_VDSO_H diff --git a/libc/src/sys/auxv/getauxval.h b/libc/src/sys/auxv/getauxval.h index 3e6971340bbef1..d9da45ff083981 100644 --- a/libc/src/sys/auxv/getauxval.h +++ b/libc/src/sys/auxv/getauxval.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_SRC_SYS_AUXV_GETAUXVAL_H #define LLVM_LIBC_SRC_SYS_AUXV_GETAUXVAL_H +#include "hdr/sys_auxv_macros.h" #include "src/__support/macros/config.h" -#include namespace LIBC_NAMESPACE_DECL { diff --git a/libc/test/src/__support/OSUtil/linux/CMakeLists.txt b/libc/test/src/__support/OSUtil/linux/CMakeLists.txt index bfb072c03e9712..ff82616cc4a701 100644 --- a/libc/test/src/__support/OSUtil/linux/CMakeLists.txt +++ b/libc/test/src/__support/OSUtil/linux/CMakeLists.txt @@ -1,3 +1,21 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE}) add_subdirectory(${LIBC_TARGET_ARCHITECTURE}) endif() + +add_libc_test( + vdso_test + SUITE libc-osutil-tests + SRCS vdso_test.cpp + DEPENDS + libc.src.__support.OSUtil.linux.vdso + libc.src.__support.OSUtil.osutil + libc.hdr.types.struct_sigaction + libc.hdr.types.struct_timeval + libc.hdr.types.struct_timespec + libc.hdr.types.clockid_t + libc.hdr.types.time_t + libc.hdr.time_macros + libc.hdr.signal_macros + libc.src.signal.sigaction + libc.src.signal.raise +) diff --git a/libc/test/src/__support/OSUtil/linux/vdso_test.cpp b/libc/test/src/__support/OSUtil/linux/vdso_test.cpp new file mode 100644 index 00000000000000..2363db69c02f97 --- /dev/null +++ b/libc/test/src/__support/OSUtil/linux/vdso_test.cpp @@ -0,0 +1,162 @@ +//===-- Unittests for VDSO ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "hdr/signal_macros.h" +#include "hdr/time_macros.h" +#include "hdr/types/clockid_t.h" +#include "hdr/types/struct_sigaction.h" +#include "hdr/types/struct_timespec.h" +#include "hdr/types/struct_timeval.h" +#include "hdr/types/time_t.h" +#include "src/__support/OSUtil/linux/vdso.h" +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/macros/properties/architectures.h" +#include "src/signal/raise.h" +#include "src/signal/sigaction.h" +#include "test/UnitTest/ErrnoSetterMatcher.h" +#include "test/UnitTest/LibcTest.h" +#include "test/UnitTest/Test.h" +#include +#include + +struct riscv_hwprobe { + int64_t key; + uint64_t value; +}; + +namespace LIBC_NAMESPACE_DECL { +// For x86_64, we explicitly test some traditional vdso symbols are indeed +// available. + +TEST(LlvmLibcOSUtilVDSOTest, GetTimeOfDay) { + vdso::TypedSymbol symbol; + if (!symbol) + return; + timeval tv; + EXPECT_EQ(symbol(&tv, nullptr), 0); + // hopefully people are not building time machines using our libc. + EXPECT_GT(tv.tv_sec, static_cast(0)); +} + +TEST(LlvmLibcOSUtilVDSOTest, Time) { + vdso::TypedSymbol symbol; + if (!symbol) + return; + time_t a, b; + EXPECT_GT(symbol(&a), static_cast(0)); + EXPECT_GT(symbol(&b), static_cast(0)); + EXPECT_GE(b, a); +} + +TEST(LlvmLibcOSUtilVDSOTest, ClockGetTime) { + vdso::TypedSymbol symbol; + if (!symbol) + return; + timespec a, b; + EXPECT_EQ(symbol(CLOCK_MONOTONIC, &a), 0); + EXPECT_EQ(symbol(CLOCK_MONOTONIC, &b), 0); + if (a.tv_sec == b.tv_sec) { + EXPECT_LT(a.tv_nsec, b.tv_nsec); + } else { + EXPECT_LT(a.tv_sec, b.tv_sec); + } +} + +TEST(LlvmLibcOSUtilVDSOTest, ClockGetTime64) { + vdso::TypedSymbol symbol; + if (!symbol) + return; + // See kernel API at + // https://elixir.bootlin.com/linux/latest/source/tools/testing/selftests/vDSO/vdso_test_correctness.c#L155 + __kernel_timespec a, b; + EXPECT_EQ(symbol(CLOCK_MONOTONIC, &a), 0); + EXPECT_EQ(symbol(CLOCK_MONOTONIC, &b), 0); + if (a.tv_sec == b.tv_sec) { + EXPECT_LT(a.tv_nsec, b.tv_nsec); + } else { + EXPECT_LT(a.tv_sec, b.tv_sec); + } +} + +TEST(LlvmLibcOSUtilVDSOTest, ClockGetRes) { + vdso::TypedSymbol symbol; + if (!symbol) + return; + timespec res{}; + EXPECT_EQ(symbol(CLOCK_MONOTONIC, &res), 0); + EXPECT_TRUE(res.tv_sec > 0 || res.tv_nsec > 0); +} + +TEST(LlvmLibcOSUtilVDSOTest, GetCpu) { + // The kernel system call has a third argument, which should be passed as + // nullptr. + vdso::TypedSymbol symbol; + if (!symbol) + return; + unsigned cpu = static_cast(-1), node = static_cast(-1); + EXPECT_EQ(symbol(&cpu, &node, nullptr), 0); + EXPECT_GE(cpu, 0u); + EXPECT_GE(node, 0u); +} + +static bool flag = false; +static void sigprof_handler [[gnu::used]] (int) { flag = true; } + +TEST(LlvmLibcOSUtilVDSOTest, RtSigReturn) { + using namespace testing::ErrnoSetterMatcher; + // must use struct since there is a function of the same name in the same + // scope. + struct sigaction sa {}; + struct sigaction old_sa {}; + sa.sa_handler = sigprof_handler; + sa.sa_flags = SA_RESTORER; + vdso::TypedSymbol symbol; + if (!symbol) + return; + sa.sa_restorer = symbol; + ASSERT_THAT(LIBC_NAMESPACE::sigaction(SIGPROF, &sa, &old_sa), Succeeds()); + raise(SIGPROF); + ASSERT_TRUE(flag); + flag = false; + ASSERT_THAT(LIBC_NAMESPACE::sigaction(SIGPROF, &old_sa, nullptr), Succeeds()); +} + +TEST(LlvmLibcOSUtilVDSOTest, FlushICache) { + vdso::TypedSymbol symbol; + if (!symbol) + return; + char buf[512]; + // we just check that the flush will not panic the program. + // the flags part only take 0/1 as up to kernel 6.10, which is used to + // indicate whether the flush is local to the core or global. + symbol(buf, buf + sizeof(buf), 0); + symbol(buf, buf + sizeof(buf), 1); +} + +// https://docs.kernel.org/6.5/riscv/hwprobe.html +TEST(LlvmLibcOSUtilVDSOTest, RiscvHwProbe) { + using namespace testing::ErrnoSetterMatcher; + vdso::TypedSymbol symbol; + if (!symbol) + return; + // If a key is unknown to the kernel, its key field will be cleared to -1, and + // its value set to 0. We expect probes.value are all 0. + // Usermode can supply NULL for cpus and 0 for cpu_count as a shortcut for all + // online CPUs + riscv_hwprobe probes[2] = {{-1, 1}, {-1, 1}}; + ASSERT_THAT(symbol(/*pairs=*/probes, /*count=*/2, /*cpusetsize=*/0, + /*cpuset=*/nullptr, + /*flags=*/0), + Succeeds()); + for (auto &probe : probes) { + EXPECT_EQ(probe.key, static_cast(-1)); + EXPECT_EQ(probe.value, static_cast(0)); + } +} + +} // namespace LIBC_NAMESPACE_DECL