14 changes: 7 additions & 7 deletions libc/src/__support/macros/cpu_features.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,31 +13,31 @@
#define LLVM_LIBC_SRC_SUPPORT_MACROS_CPU_FEATURES_H

#if defined(__SSE2__)
#define LIBC_TARGET_HAS_SSE2
#define LIBC_TARGET_CPU_HAS_SSE2
#endif

#if defined(__SSE4_2__)
#define LIBC_TARGET_HAS_SSE4_2
#define LIBC_TARGET_CPU_HAS_SSE4_2
#endif

#if defined(__AVX__)
#define LIBC_TARGET_HAS_AVX
#define LIBC_TARGET_CPU_HAS_AVX
#endif

#if defined(__AVX2__)
#define LIBC_TARGET_HAS_AVX2
#define LIBC_TARGET_CPU_HAS_AVX2
#endif

#if defined(__AVX512F__)
#define LIBC_TARGET_HAS_AVX512F
#define LIBC_TARGET_CPU_HAS_AVX512F
#endif

#if defined(__AVX512BW__)
#define LIBC_TARGET_HAS_AVX512BW
#define LIBC_TARGET_CPU_HAS_AVX512BW
#endif

#if defined(__ARM_FEATURE_FMA) || defined(__AVX2__) || defined(__FMA__)
#define LIBC_TARGET_HAS_FMA
#define LIBC_TARGET_CPU_HAS_FMA
#endif

#endif // LLVM_LIBC_SRC_SUPPORT_MACROS_CPU_FEATURES_H
16 changes: 14 additions & 2 deletions libc/src/__support/macros/sanitizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,20 @@

#if LIBC_HAVE_MEMORY_SANITIZER
#include <sanitizer/msan_interface.h>
#define SANITIZER_MEMORY_INITIALIZED(addr, size) __msan_unpoison(addr, size)
#define MSAN_UNPOISON(addr, size) __msan_unpoison(addr, size)
#else
#define SANITIZER_MEMORY_INITIALIZED(ptr, size)
#define MSAN_UNPOISON(ptr, size)
#endif

#if LIBC_HAVE_ADDRESS_SANITIZER
#include <sanitizer/asan_interface.h>
#define ASAN_POISON_MEMORY_REGION(addr, size) \
__asan_poison_memory_region((addr), (size))
#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
__asan_unpoison_memory_region((addr), (size))
#else
#define ASAN_POISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))
#define ASAN_UNPOISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))
#endif

#endif // LLVM_LIBC_SRC_SUPPORT_MACROS_SANITIZER_H
12 changes: 6 additions & 6 deletions libc/src/__support/threads/linux/thread.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#include "src/__support/error_or.h"
#include "src/__support/threads/linux/futex_word.h" // For FutexWordType

#ifdef LIBC_TARGET_IS_AARCH64
#ifdef LIBC_TARGET_ARCH_IS_AARCH64
#include <arm_acle.h>
#endif

Expand Down Expand Up @@ -99,14 +99,14 @@ __attribute__((always_inline)) inline uintptr_t get_start_args_addr() {
// NOTE: For __builtin_frame_address to work reliably across compilers,
// architectures and various optimization levels, the TU including this file
// should be compiled with -fno-omit-frame-pointer.
#ifdef LIBC_TARGET_IS_X86_64
#ifdef LIBC_TARGET_ARCH_IS_X86_64
return reinterpret_cast<uintptr_t>(__builtin_frame_address(0))
// The x86_64 call instruction pushes resume address on to the stack.
// Next, The x86_64 SysV ABI requires that the frame pointer be pushed
// on to the stack. So, we have to step past two 64-bit values to get
// to the start args.
+ sizeof(uintptr_t) * 2;
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
// The frame pointer after cloning the new thread in the Thread::run method
// is set to the stack pointer where start args are stored. So, we fetch
// from there.
Expand Down Expand Up @@ -190,15 +190,15 @@ int Thread::run(ThreadStyle style, ThreadRunner runner, void *arg, void *stack,
// Also, we want the result of the syscall to be in a register as the child
// thread gets a completely different stack after it is created. The stack
// variables from this function will not be availalbe to the child thread.
#ifdef LIBC_TARGET_IS_X86_64
#ifdef LIBC_TARGET_ARCH_IS_X86_64
long register clone_result asm("rax");
clone_result = __llvm_libc::syscall_impl(
SYS_clone, CLONE_SYSCALL_FLAGS, adjusted_stack,
&attrib->tid, // The address where the child tid is written
&clear_tid->val, // The futex where the child thread status is signalled
tls.tp // The thread pointer value for the new thread.
);
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
long register clone_result asm("x0");
clone_result = __llvm_libc::syscall_impl(
SYS_clone, CLONE_SYSCALL_FLAGS, adjusted_stack,
Expand All @@ -211,7 +211,7 @@ int Thread::run(ThreadStyle style, ThreadRunner runner, void *arg, void *stack,
#endif

if (clone_result == 0) {
#ifdef LIBC_TARGET_IS_AARCH64
#ifdef LIBC_TARGET_ARCH_IS_AARCH64
// We set the frame pointer to be the same as the "sp" so that start args
// can be sniffed out from start_thread.
__arm_wsr64("x29", __arm_rsr64("sp"));
Expand Down
3 changes: 2 additions & 1 deletion libc/src/__support/threads/thread.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ union ThreadReturnValue {
constexpr ThreadReturnValue(void *r) : posix_retval(r) {}
};

#if (defined(LIBC_TARGET_IS_AARCH64) || defined(LIBC_TARGET_IS_X86_64))
#if (defined(LIBC_TARGET_ARCH_IS_AARCH64) || \
defined(LIBC_TARGET_ARCH_IS_X86_64))
constexpr unsigned int STACK_ALIGNMENT = 16;
#endif
// TODO: Provide stack alignment requirements for other architectures.
Expand Down
4 changes: 2 additions & 2 deletions libc/src/math/generic/asinf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,12 @@ LLVM_LIBC_FUNCTION(float, asinf, (float x)) {
// |x| < 2^-125. For targets without FMA instructions, we simply use
// double for intermediate results as it is more efficient than using an
// emulated version of FMA.
#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
return fputil::multiply_add(x, 0x1.0p-25f, x);
#else
double xd = static_cast<double>(x);
return static_cast<float>(fputil::multiply_add(xd, 0x1.0p-25, xd));
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA
}

// Check for exceptional values
Expand Down
4 changes: 2 additions & 2 deletions libc/src/math/generic/cosf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,11 @@ LLVM_LIBC_FUNCTION(float, cosf, (float x)) {
// |x| < 2^-125. For targets without FMA instructions, we simply use
// double for intermediate results as it is more efficient than using an
// emulated version of FMA.
#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
#else
return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA
}

if (auto r = COSF_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
Expand Down
8 changes: 4 additions & 4 deletions libc/src/math/generic/expm1f.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,14 @@ LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
return 0x1.8dbe62p-3f;
}

#if !defined(LIBC_TARGET_HAS_FMA)
#if !defined(LIBC_TARGET_CPU_HAS_FMA)
if (LIBC_UNLIKELY(x_u == 0xbdc1'c6cbU)) { // x = -0x1.838d96p-4f
int round_mode = fputil::get_round();
if (round_mode == FE_TONEAREST || round_mode == FE_DOWNWARD)
return -0x1.71c884p-4f;
return -0x1.71c882p-4f;
}
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA

// When |x| > 25*log(2), or nan
if (LIBC_UNLIKELY(x_abs >= 0x418a'a123U)) {
Expand Down Expand Up @@ -100,12 +100,12 @@ LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
// 2^-76. For targets without FMA instructions, we simply use double for
// intermediate results as it is more efficient than using an emulated
// version of FMA.
#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
return fputil::fma(x, x, x);
#else
double xd = x;
return static_cast<float>(fputil::multiply_add(xd, xd, xd));
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA
}

// 2^-25 <= |x| < 2^-4
Expand Down
4 changes: 2 additions & 2 deletions libc/src/math/generic/sincosf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,14 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) {
// |x| < 2^-125. For targets without FMA instructions, we simply use
// double for intermediate results as it is more efficient than using an
// emulated version of FMA.
#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
*sinp = fputil::multiply_add(x, -0x1.0p-25f, x);
*cosp = fputil::multiply_add(FPBits(x_abs).get_val(), -0x1.0p-25f, 1.0f);
#else
*sinp = static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, xd));
*cosp = static_cast<float>(fputil::multiply_add(
static_cast<double>(FPBits(x_abs).get_val()), -0x1.0p-25, 1.0));
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA
return;
}

Expand Down
4 changes: 2 additions & 2 deletions libc/src/math/generic/sincosf_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
#include "src/__support/common.h"
#include "src/__support/macros/cpu_features.h"

#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
#include "range_reduction_fma.h"
// using namespace __llvm_libc::fma;
using __llvm_libc::fma::FAST_PASS_BOUND;
Expand All @@ -26,7 +26,7 @@ using __llvm_libc::fma::small_range_reduction;
using __llvm_libc::generic::FAST_PASS_BOUND;
using __llvm_libc::generic::large_range_reduction;
using __llvm_libc::generic::small_range_reduction;
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA

namespace __llvm_libc {

Expand Down
6 changes: 3 additions & 3 deletions libc/src/math/generic/sinf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

#include <errno.h>

#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
#include "range_reduction_fma.h"
#else
#include "range_reduction.h"
Expand Down Expand Up @@ -100,11 +100,11 @@ LLVM_LIBC_FUNCTION(float, sinf, (float x)) {
// |x| < 2^-125. For targets without FMA instructions, we simply use
// double for intermediate results as it is more efficient than using an
// emulated version of FMA.
#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
return fputil::multiply_add(x, -0x1.0p-25f, x);
#else
return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, xd));
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA
}

// |x| < pi/16.
Expand Down
4 changes: 2 additions & 2 deletions libc/src/math/generic/tanf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,11 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) {
// |x| < 2^-125. For targets without FMA instructions, we simply use
// double for intermediate results as it is more efficient than using an
// emulated version of FMA.
#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
return fputil::multiply_add(x, 0x1.0p-25f, x);
#else
return static_cast<float>(fputil::multiply_add(xd, 0x1.0p-25, xd));
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA
}

// |x| < pi/32
Expand Down
4 changes: 2 additions & 2 deletions libc/src/math/generic/tanhf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,13 @@ LLVM_LIBC_FUNCTION(float, tanhf, (float x)) {
auto ep = exp_b_range_reduc<ExpBase>(2.0f * x); // exp(2 * x)
double r = ExpBase::powb_lo(ep.lo);
// tanh(x) = (exp(2x) - 1) / (exp(2x) + 1)
#if defined(LIBC_TARGET_HAS_FMA)
#if defined(LIBC_TARGET_CPU_HAS_FMA)
return fputil::multiply_add(ep.mh, r, -1.0) /
fputil::multiply_add(ep.mh, r, 1.0);
#else
double exp_x = ep.mh * r;
return (exp_x - 1.0) / (exp_x + 1.0);
#endif // LIBC_TARGET_HAS_FMA
#endif // LIBC_TARGET_CPU_HAS_FMA
}

} // namespace __llvm_libc
4 changes: 2 additions & 2 deletions libc/src/setjmp/longjmp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
namespace __llvm_libc {

LLVM_LIBC_FUNCTION(void, longjmp, (__jmp_buf * buf, int val)) {
#ifdef LIBC_TARGET_IS_X86_64
#ifdef LIBC_TARGET_ARCH_IS_X86_64
register __UINT64_TYPE__ rbx __asm__("rbx");
register __UINT64_TYPE__ rbp __asm__("rbp");
register __UINT64_TYPE__ r12 __asm__("r12");
Expand All @@ -38,7 +38,7 @@ LLVM_LIBC_FUNCTION(void, longjmp, (__jmp_buf * buf, int val)) {
LIBC_INLINE_ASM("mov %1, %0\n\t" : "=r"(r15) : "m"(buf->r15) :);
LIBC_INLINE_ASM("mov %1, %0\n\t" : "=r"(rsp) : "m"(buf->rsp) :);
LIBC_INLINE_ASM("jmp *%0\n\t" : : "m"(buf->rip));
#else // LIBC_TARGET_IS_X86_64
#else // LIBC_TARGET_ARCH_IS_X86_64
#error "longjmp implementation not available for the target architecture."
#endif
}
Expand Down
4 changes: 2 additions & 2 deletions libc/src/setjmp/setjmp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
namespace __llvm_libc {

LLVM_LIBC_FUNCTION(int, setjmp, (__jmp_buf * buf)) {
#ifdef LIBC_TARGET_IS_X86_64
#ifdef LIBC_TARGET_ARCH_IS_X86_64
register __UINT64_TYPE__ rbx __asm__("rbx");
register __UINT64_TYPE__ r12 __asm__("r12");
register __UINT64_TYPE__ r13 __asm__("r13");
Expand Down Expand Up @@ -50,7 +50,7 @@ LLVM_LIBC_FUNCTION(int, setjmp, (__jmp_buf * buf)) {
buf->rsp = reinterpret_cast<__UINTPTR_TYPE__>(__builtin_frame_address(0)) +
sizeof(__UINTPTR_TYPE__) * 2;
buf->rip = reinterpret_cast<__UINTPTR_TYPE__>(__builtin_return_address(0));
#else // LIBC_TARGET_IS_X86_64
#else // LIBC_TARGET_ARCH_IS_X86_64
#error "setjmp implementation not available for the target architecture."
#endif

Expand Down
21 changes: 11 additions & 10 deletions libc/src/string/memory_utils/bcmp_implementations.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ inline_bcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) {
return BcmpReturnType::ZERO();
}

#if defined(LIBC_TARGET_IS_X86) || defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
[[maybe_unused]] LIBC_INLINE BcmpReturnType
inline_bcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
if (count < 256)
Expand All @@ -39,9 +39,10 @@ inline_bcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
align_to_next_boundary<64, Arg::P1>(p1, p2, count);
return generic::Bcmp<64>::loop_and_tail(p1, p2, count);
}
#endif // defined(LIBC_TARGET_IS_X86) || defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_X86) ||
// defined(LIBC_TARGET_ARCH_IS_AARCH64)

#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
[[maybe_unused]] LIBC_INLINE BcmpReturnType
inline_bcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) {
if (count <= 32)
Expand Down Expand Up @@ -109,9 +110,9 @@ inline_bcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) {
else
return inline_bcmp_generic_gt16(p1, p2, count);
}
#endif // defined(LIBC_TARGET_IS_X86)
#endif // defined(LIBC_TARGET_ARCH_IS_X86)

#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_aarch64(CPtr p1,
CPtr p2,
size_t count) {
Expand Down Expand Up @@ -158,16 +159,16 @@ inline_bcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) {
}
return aarch64::Bcmp<32>::loop_and_tail(p1, p2, count);
}
#endif // defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)

LIBC_INLINE BcmpReturnType inline_bcmp(CPtr p1, CPtr p2, size_t count) {
#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
return inline_bcmp_x86(p1, p2, count);
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
return inline_bcmp_aarch64(p1, p2, count);
#elif defined(LIBC_TARGET_IS_ARM)
#elif defined(LIBC_TARGET_ARCH_IS_ARM)
return inline_bcmp_embedded_tiny(p1, p2, count);
#elif defined(LIBC_TARGET_IS_GPU)
#elif defined(LIBC_TARGET_ARCH_IS_GPU)
return inline_bcmp_embedded_tiny(p1, p2, count);
#else
#error "Unsupported platform"
Expand Down
23 changes: 12 additions & 11 deletions libc/src/string/memory_utils/memcmp_implementations.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ inline_memcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) {
return MemcmpReturnType::ZERO();
}

#if defined(LIBC_TARGET_IS_X86) || defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
[[maybe_unused]] LIBC_INLINE MemcmpReturnType
inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
if (LIBC_UNLIKELY(count >= 384)) {
Expand All @@ -39,9 +39,10 @@ inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
}
return generic::Memcmp<16>::loop_and_tail(p1, p2, count);
}
#endif // defined(LIBC_TARGET_IS_X86) || defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_X86) ||
// defined(LIBC_TARGET_ARCH_IS_AARCH64)

#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
[[maybe_unused]] LIBC_INLINE MemcmpReturnType
inline_memcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) {
if (LIBC_UNLIKELY(count >= 384)) {
Expand Down Expand Up @@ -84,9 +85,9 @@ inline_memcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) {
return x86::avx512bw::Memcmp<64>::loop_and_tail(p1, p2, count);
}

#endif // defined(LIBC_TARGET_IS_X86)
#endif // defined(LIBC_TARGET_ARCH_IS_X86)

#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
[[maybe_unused]] LIBC_INLINE MemcmpReturnType
inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) {
if (LIBC_UNLIKELY(count >= 128)) { // [128, ∞]
Expand All @@ -106,10 +107,10 @@ inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) {
// [64, 127]
return generic::Memcmp<16>::loop_and_tail(p1 + 32, p2 + 32, count - 32);
}
#endif // defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)

LIBC_INLINE MemcmpReturnType inline_memcmp(CPtr p1, CPtr p2, size_t count) {
#if defined(LIBC_TARGET_IS_X86) || defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
if (count == 0)
return MemcmpReturnType::ZERO();
if (count == 1)
Expand All @@ -122,7 +123,7 @@ LIBC_INLINE MemcmpReturnType inline_memcmp(CPtr p1, CPtr p2, size_t count) {
return generic::Memcmp<4>::head_tail(p1, p2, count);
if (count <= 16)
return generic::Memcmp<8>::head_tail(p1, p2, count);
#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
if constexpr (x86::kAvx512BW)
return inline_memcmp_x86_avx512bw_gt16(p1, p2, count);
else if constexpr (x86::kAvx2)
Expand All @@ -131,15 +132,15 @@ LIBC_INLINE MemcmpReturnType inline_memcmp(CPtr p1, CPtr p2, size_t count) {
return inline_memcmp_x86_sse2_gt16(p1, p2, count);
else
return inline_memcmp_generic_gt16(p1, p2, count);
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
if constexpr (aarch64::kNeon)
return inline_memcmp_aarch64_neon_gt16(p1, p2, count);
else
return inline_memcmp_generic_gt16(p1, p2, count);
#endif
#elif defined(LIBC_TARGET_IS_ARM)
#elif defined(LIBC_TARGET_ARCH_IS_ARM)
return inline_memcmp_embedded_tiny(p1, p2, count);
#elif defined(LIBC_TARGET_IS_GPU)
#elif defined(LIBC_TARGET_ARCH_IS_GPU)
return inline_memcmp_embedded_tiny(p1, p2, count);
#else
#error "Unsupported platform"
Expand Down
16 changes: 8 additions & 8 deletions libc/src/string/memory_utils/memcpy_implementations.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src,
builtin::Memcpy<1>::block(dst + offset, src + offset);
}

#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
if (count == 0)
Expand Down Expand Up @@ -86,9 +86,9 @@ inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
return inline_memcpy_x86(dst, src, count);
}
}
#endif // defined(LIBC_TARGET_IS_X86)
#endif // defined(LIBC_TARGET_ARCH_IS_X86)

#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
if (count == 0)
Expand All @@ -115,18 +115,18 @@ inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
align_to_next_boundary<16, Arg::Src>(dst, src, count);
return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
}
#endif // defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)

LIBC_INLINE void inline_memcpy(Ptr __restrict dst, CPtr __restrict src,
size_t count) {
using namespace __llvm_libc::builtin;
#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
return inline_memcpy_x86_maybe_interpose_repmovsb(dst, src, count);
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
return inline_memcpy_aarch64(dst, src, count);
#elif defined(LIBC_TARGET_IS_ARM)
#elif defined(LIBC_TARGET_ARCH_IS_ARM)
return inline_memcpy_embedded_tiny(dst, src, count);
#elif defined(LIBC_TARGET_IS_GPU)
#elif defined(LIBC_TARGET_ARCH_IS_GPU)
return inline_memcpy_embedded_tiny(dst, src, count);
#else
#error "Unsupported platform"
Expand Down
10 changes: 5 additions & 5 deletions libc/src/string/memory_utils/memmove_implementations.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,13 @@ template <size_t MaxSize>
}

LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) {
#if defined(LIBC_TARGET_IS_X86) || defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_X86)
static constexpr size_t kMaxSize = x86::kAvx512F ? 64
: x86::kAvx ? 32
: x86::kSse2 ? 16
: 8;
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8;
#endif
// return inline_memmove_generic<kMaxSize>(dst, src, count);
Expand Down Expand Up @@ -101,9 +101,9 @@ LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) {
return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src,
count);
}
#elif defined(LIBC_TARGET_IS_ARM)
#elif defined(LIBC_TARGET_ARCH_IS_ARM)
return inline_memmove_embedded_tiny(dst, src, count);
#elif defined(LIBC_TARGET_IS_GPU)
#elif defined(LIBC_TARGET_ARCH_IS_GPU)
return inline_memmove_embedded_tiny(dst, src, count);
#else
#error "Unsupported platform"
Expand Down
16 changes: 8 additions & 8 deletions libc/src/string/memory_utils/memset_implementations.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ inline_memset_embedded_tiny(Ptr dst, uint8_t value, size_t count) {
generic::Memset<1, 1>::block(dst + offset, value);
}

#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
template <size_t MaxSize>
[[maybe_unused]] LIBC_INLINE static void
inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
Expand All @@ -55,9 +55,9 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
align_to_next_boundary<32>(dst, count);
return generic::Memset<32, MaxSize>::loop_and_tail(dst, value, count);
}
#endif // defined(LIBC_TARGET_IS_X86)
#endif // defined(LIBC_TARGET_ARCH_IS_X86)

#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
template <size_t MaxSize>
[[maybe_unused]] LIBC_INLINE static void
inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
Expand Down Expand Up @@ -93,21 +93,21 @@ inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
return generic::Memset<64, MaxSize>::loop_and_tail(dst, value, count);
}
}
#endif // defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)

LIBC_INLINE static void inline_memset(Ptr dst, uint8_t value, size_t count) {
#if defined(LIBC_TARGET_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_X86)
static constexpr size_t kMaxSize = x86::kAvx512F ? 64
: x86::kAvx ? 32
: x86::kSse2 ? 16
: 8;
return inline_memset_x86<kMaxSize>(dst, value, count);
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8;
return inline_memset_aarch64<kMaxSize>(dst, value, count);
#elif defined(LIBC_TARGET_IS_ARM)
#elif defined(LIBC_TARGET_ARCH_IS_ARM)
return inline_memset_embedded_tiny(dst, value, count);
#elif defined(LIBC_TARGET_IS_GPU)
#elif defined(LIBC_TARGET_ARCH_IS_GPU)
return inline_memset_embedded_tiny(dst, value, count);
#else
#error "Unsupported platform"
Expand Down
4 changes: 2 additions & 2 deletions libc/src/string/memory_utils/op_aarch64.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

#include "src/__support/macros/architectures.h"

#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)

#include "src/__support/common.h"
#include "src/string/memory_utils/op_generic.h"
Expand Down Expand Up @@ -174,6 +174,6 @@ template <size_t Size> struct Bcmp {

} // namespace __llvm_libc::aarch64

#endif // LIBC_TARGET_IS_AARCH64
#endif // LIBC_TARGET_ARCH_IS_AARCH64

#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H
2 changes: 1 addition & 1 deletion libc/src/string/memory_utils/op_generic.h
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ static_assert((UINTPTR_MAX == 4294967295U) ||
(UINTPTR_MAX == 18446744073709551615UL),
"We currently only support 32- or 64-bit platforms");

#if defined(LIBC_TARGET_IS_X86_64) || defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_X86_64) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
#define LLVM_LIBC_HAS_UINT64
#endif

Expand Down
4 changes: 2 additions & 2 deletions libc/src/string/memory_utils/op_x86.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

#include "src/__support/macros/architectures.h"

#if defined(LIBC_TARGET_IS_X86_64)
#if defined(LIBC_TARGET_ARCH_IS_X86_64)

#include "src/__support/common.h"
#include "src/string/memory_utils/op_builtin.h"
Expand Down Expand Up @@ -269,6 +269,6 @@ template <size_t Size> using Memcmp = MemcmpImpl<Size, 64, memcmp64, bcmp64>;

} // namespace __llvm_libc::x86

#endif // LIBC_TARGET_IS_X86_64
#endif // LIBC_TARGET_ARCH_IS_X86_64

#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_X86_H
3 changes: 2 additions & 1 deletion libc/src/threads/linux/Futex.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@

namespace __llvm_libc {

#if (defined(LIBC_TARGET_IS_AARCH64) || defined(LIBC_TARGET_IS_X86_64))
#if (defined(LIBC_TARGET_ARCH_IS_AARCH64) || \
defined(LIBC_TARGET_ARCH_IS_X86_64))
// The futex data has to be exactly 4 bytes long. However, we use a uint type
// here as we do not want to use `uint32_t` type to match the public definitions
// of types which include a field for a futex word. With public definitions, we
Expand Down
4 changes: 2 additions & 2 deletions libc/test/src/fenv/enabled_exceptions_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
// This test enables an exception and verifies that raising that exception
// triggers SIGFPE.
TEST(LlvmLibcExceptionStatusTest, RaiseAndCrash) {
#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
// Few aarch64 HW implementations do not trap exceptions. We skip this test
// completely on such HW.
//
Expand All @@ -33,7 +33,7 @@ TEST(LlvmLibcExceptionStatusTest, RaiseAndCrash) {
__llvm_libc::fputil::enable_except(FE_DIVBYZERO);
if (__llvm_libc::fputil::get_except() == 0)
return;
#endif // defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)

// TODO: Install a floating point exception handler and verify that the
// the expected exception was raised. One will have to longjmp back from
Expand Down
4 changes: 2 additions & 2 deletions libc/test/src/fenv/feenableexcept_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#include <fenv.h>

TEST(LlvmLibcFEnvTest, EnableTest) {
#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
// Few aarch64 HW implementations do not trap exceptions. We skip this test
// completely on such HW.
//
Expand All @@ -28,7 +28,7 @@ TEST(LlvmLibcFEnvTest, EnableTest) {
__llvm_libc::feenableexcept(FE_DIVBYZERO);
if (__llvm_libc::fegetexcept() == 0)
return;
#endif // defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)

int excepts[] = {FE_DIVBYZERO, FE_INVALID, FE_INEXACT, FE_OVERFLOW,
FE_UNDERFLOW};
Expand Down
4 changes: 2 additions & 2 deletions libc/test/src/fenv/feholdexcept_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#include <fenv.h>

TEST(LlvmLibcFEnvTest, RaiseAndCrash) {
#if defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
// Few aarch64 HW implementations do not trap exceptions. We skip this test
// completely on such HW.
//
Expand All @@ -28,7 +28,7 @@ TEST(LlvmLibcFEnvTest, RaiseAndCrash) {
__llvm_libc::fputil::enable_except(FE_DIVBYZERO);
if (__llvm_libc::fputil::get_except() == 0)
return;
#endif // defined(LIBC_TARGET_IS_AARCH64)
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)

int excepts[] = {FE_DIVBYZERO, FE_INVALID, FE_INEXACT, FE_OVERFLOW,
FE_UNDERFLOW};
Expand Down
1 change: 1 addition & 0 deletions libc/test/src/string/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ function(add_libc_multi_impl_test name)
${ARGN}
DEPENDS
${fq_config_name}
libc.src.__support.sanitizer
)
get_fq_target_name(${fq_config_name}_test fq_target_name)
else()
Expand Down
12 changes: 1 addition & 11 deletions libc/test/src/string/memory_utils/memory_check_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,23 +11,13 @@

#include "src/__support/CPP/span.h"
#include "src/__support/macros/compiler_features.h"
#include "src/__support/macros/sanitizer.h"
#include "src/string/memory_utils/utils.h"
#include <assert.h> // assert
#include <stddef.h> // size_t
#include <stdint.h> // uintxx_t
#include <stdlib.h> // malloc/free

#if LIBC_HAS_FEATURE(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
#include <sanitizer/asan_interface.h>
#define ASAN_POISON_MEMORY_REGION(addr, size) \
__asan_poison_memory_region((addr), (size))
#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
__asan_unpoison_memory_region((addr), (size))
#else
#define ASAN_POISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))
#define ASAN_UNPOISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))
#endif

namespace __llvm_libc {

// Simple structure to allocate a buffer of a particular size.
Expand Down
4 changes: 2 additions & 2 deletions libc/test/src/string/memory_utils/op_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

#include <assert.h>

#if defined(LIBC_TARGET_IS_X86_64) || defined(LIBC_TARGET_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_X86_64) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
#define LLVM_LIBC_HAS_UINT64
#endif

Expand Down Expand Up @@ -200,7 +200,7 @@ using BcmpImplementations = testing::TypeList<
x86::avx512bw::Bcmp<64>, //
x86::avx512bw::Bcmp<128>, //
#endif
#ifdef LIBC_TARGET_IS_AARCH64
#ifdef LIBC_TARGET_ARCH_IS_AARCH64
aarch64::Bcmp<16>, //
aarch64::Bcmp<32>, //
#endif
Expand Down
4 changes: 2 additions & 2 deletions libc/test/src/sys/utsname/uname_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
TEST(LlvmLibcUnameTest, GetMachineName) {
struct utsname names;
ASSERT_GE(__llvm_libc::uname(&names), 0);
#ifdef LIBC_TARGET_IS_X86_64
#ifdef LIBC_TARGET_ARCH_IS_X86_64
ASSERT_STREQ(names.machine, "x86_64");
#elif defined(LIBC_TARGET_IS_AARCH64)
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
ASSERT_STREQ(names.machine, "aarch64");
#endif
}
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ cc_library(
hdrs = ["memory_utils/memory_check_utils.h"],
deps = [
"//libc:__support_cpp_span",
"//libc:__support_sanitizer",
"//libc:string_memory_utils",
],
)
Expand Down