diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt index 8b65a8839ab21..a389a6d1702fe 100644 --- a/libc/src/__support/CPP/CMakeLists.txt +++ b/libc/src/__support/CPP/CMakeLists.txt @@ -210,3 +210,9 @@ add_object_library( libc.src.__support.common libc.src.__support.macros.properties.os ) + +add_header_library( + simd + HDRS + simd.h +) diff --git a/libc/src/__support/CPP/algorithm.h b/libc/src/__support/CPP/algorithm.h index 7704b3fa81f0c..de0c47369d945 100644 --- a/libc/src/__support/CPP/algorithm.h +++ b/libc/src/__support/CPP/algorithm.h @@ -18,6 +18,12 @@ namespace LIBC_NAMESPACE_DECL { namespace cpp { +template struct plus {}; +template struct multiplies {}; +template struct bit_and {}; +template struct bit_or {}; +template struct bit_xor {}; + template LIBC_INLINE constexpr const T &max(const T &a, const T &b) { return (a < b) ? b : a; } diff --git a/libc/src/__support/CPP/simd.h b/libc/src/__support/CPP/simd.h new file mode 100644 index 0000000000000..972f156419d4d --- /dev/null +++ b/libc/src/__support/CPP/simd.h @@ -0,0 +1,227 @@ +//===-- Portable SIMD library similar to stdx::simd -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a generic interface into fixed-size SIMD instructions +// using the clang vector type. The API shares some similarities with the +// stdx::simd proposal, but instead chooses to use vectors as primitive types +// with several extra helper functions. +// +//===----------------------------------------------------------------------===// + +#include "hdr/stdint_proxy.h" +#include "src/__support/CPP/algorithm.h" +#include "src/__support/CPP/bit.h" +#include "src/__support/CPP/type_traits/integral_constant.h" +#include "src/__support/macros/attributes.h" +#include "src/__support/macros/config.h" + +#include + +#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_SIMD_H +#define LLVM_LIBC_SRC___SUPPORT_CPP_SIMD_H + +#if LIBC_HAS_VECTOR_TYPE + +namespace LIBC_NAMESPACE_DECL { +namespace cpp { + +namespace internal { + +template +using get_as_integer_type_t = unsigned _BitInt(sizeof(T) * CHAR_BIT); + +#if defined(LIBC_TARGET_CPU_HAS_AVX512F) +template +inline constexpr size_t native_vector_size = 64 / sizeof(T); +#elif defined(LIBC_TARGET_CPU_HAS_AVX2) +template +inline constexpr size_t native_vector_size = 32 / sizeof(T); +#elif defined(LIBC_TARGET_CPU_HAS_SSE2) || defined(LIBC_TARGET_CPU_HAS_ARM_NEON) +template +inline constexpr size_t native_vector_size = 16 / sizeof(T); +#else +template inline constexpr size_t native_vector_size = 1; +#endif + +template LIBC_INLINE constexpr T poison() { + return __builtin_nondeterministic_value(T()); +} +} // namespace internal + +// Type aliases. +template +using fixed_size_simd = T [[clang::ext_vector_type(N)]]; +template > +using simd = T [[clang::ext_vector_type(N)]]; +template +using simd_mask = simd>; + +// Type trait helpers. +template +struct simd_size : cpp::integral_constant { +}; +template constexpr size_t simd_size_v = simd_size::value; + +template struct is_simd : cpp::integral_constant {}; +template +struct is_simd> : cpp::integral_constant {}; +template constexpr bool is_simd_v = is_simd::value; + +template +struct is_simd_mask : cpp::integral_constant {}; +template +struct is_simd_mask> : cpp::integral_constant {}; +template constexpr bool is_simd_mask_v = is_simd_mask::value; + +template struct simd_element_type; +template struct simd_element_type> { + using type = T; +}; +template +using simd_element_type_t = typename simd_element_type::type; + +template +using enable_if_simd_t = cpp::enable_if_t, T>; + +// Casting. +template +LIBC_INLINE constexpr simd simd_cast(simd v) { + return __builtin_convertvector(v, simd); +} + +// SIMD mask operations. +template LIBC_INLINE constexpr bool all_of(simd m) { + return __builtin_reduce_and(m); +} +template LIBC_INLINE constexpr bool any_of(simd m) { + return __builtin_reduce_or(m); +} +template LIBC_INLINE constexpr bool none_of(simd m) { + return !any_of(m); +} +template LIBC_INLINE constexpr bool some_of(simd m) { + return any_of(m) && !all_of(m); +} +template LIBC_INLINE constexpr int popcount(simd m) { + return __builtin_popcountg(m); +} +template LIBC_INLINE constexpr int find_first_set(simd m) { + return __builtin_ctzg(m); +} +template LIBC_INLINE constexpr int find_last_set(simd m) { + constexpr size_t size = simd_size_v>; + return size - __builtin_clzg(m); +} + +// Elementwise operations. +template +LIBC_INLINE constexpr simd min(simd x, simd y) { + return __builtin_elementwise_min(x, y); +} +template +LIBC_INLINE constexpr simd max(simd x, simd y) { + return __builtin_elementwise_max(x, y); +} + +// Reduction operations. +template > +LIBC_INLINE constexpr T reduce(simd v, Op op = {}) { + return reduce(v, op); +} +template +LIBC_INLINE constexpr T reduce(simd v, cpp::plus<>) { + return __builtin_reduce_add(v); +} +template +LIBC_INLINE constexpr T reduce(simd v, cpp::multiplies<>) { + return __builtin_reduce_mul(v); +} +template +LIBC_INLINE constexpr T reduce(simd v, cpp::bit_and<>) { + return __builtin_reduce_and(v); +} +template +LIBC_INLINE constexpr T reduce(simd v, cpp::bit_or<>) { + return __builtin_reduce_or(v); +} +template +LIBC_INLINE constexpr T reduce(simd v, cpp::bit_xor<>) { + return __builtin_reduce_xor(v); +} +template LIBC_INLINE constexpr T hmin(simd v) { + return __builtin_reduce_min(v); +} +template LIBC_INLINE constexpr T hmax(simd v) { + return __builtin_reduce_max(v); +} + +// Accessor helpers. +template +LIBC_INLINE enable_if_simd_t load_unaligned(const void *ptr) { + T tmp; + __builtin_memcpy(&tmp, ptr, sizeof(T)); + return tmp; +} +template +LIBC_INLINE enable_if_simd_t load_aligned(const void *ptr) { + return load_unaligned(__builtin_assume_aligned(ptr, alignof(T))); +} +template +LIBC_INLINE enable_if_simd_t store_unaligned(T v, void *ptr) { + __builtin_memcpy(ptr, &v, sizeof(T)); +} +template +LIBC_INLINE enable_if_simd_t store_aligned(T v, void *ptr) { + store_unaligned(v, __builtin_assume_aligned(ptr, alignof(T))); +} +template +LIBC_INLINE enable_if_simd_t +masked_load(simd> m, void *ptr, + T passthru = internal::poison>()) { + return __builtin_masked_load(m, ptr, passthru); +} +template +LIBC_INLINE enable_if_simd_t masked_store(simd> m, T v, + void *ptr) { + __builtin_masked_store( + m, v, static_cast(__builtin_assume_aligned(ptr, alignof(T)))); +} + +// Construction helpers. +template LIBC_INLINE constexpr simd splat(T v) { + return simd(v); +} +template LIBC_INLINE constexpr simd splat(T v) { + return splat>>(v); +} +template +LIBC_INLINE constexpr simd iota(T base = T(0), T step = T(1)) { + simd v{}; + for (unsigned i = 0; i < N; ++i) + v[i] = base + T(i) * step; + return v; +} +template +LIBC_INLINE constexpr simd iota(T base = T(0), T step = T(1)) { + return iota>>(base, step); +} + +// Conditional helpers. +template +LIBC_INLINE constexpr simd select(simd m, simd x, + simd y) { + return m ? x : y; +} + +// TODO: where expressions, scalar overloads, ABI types. + +} // namespace cpp +} // namespace LIBC_NAMESPACE_DECL + +#endif // LIBC_HAS_VECTOR_TYPE +#endif diff --git a/libc/src/__support/macros/attributes.h b/libc/src/__support/macros/attributes.h index 4ff374b0e4fbd..d350a06125f00 100644 --- a/libc/src/__support/macros/attributes.h +++ b/libc/src/__support/macros/attributes.h @@ -73,4 +73,10 @@ LIBC_THREAD_MODE_EXTERNAL. #define LIBC_PREFERED_TYPE(TYPE) #endif +#if __has_attribute(ext_vector_type) && __has_feature(ext_vector_type_boolean) +#define LIBC_HAS_VECTOR_TYPE 1 +#else +#define LIBC_HAS_VECTOR_TYPE 0 +#endif + #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_ATTRIBUTES_H diff --git a/libc/src/__support/macros/properties/cpu_features.h b/libc/src/__support/macros/properties/cpu_features.h index fde30eadfd83b..fc6099ca6ccc5 100644 --- a/libc/src/__support/macros/properties/cpu_features.h +++ b/libc/src/__support/macros/properties/cpu_features.h @@ -59,6 +59,10 @@ #endif // LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE #endif // __ARM_FP +#if defined(__ARM_NEON) +#define LIBC_TARGET_CPU_HAS_ARM_NEON +#endif + #if defined(__riscv_flen) // https://github.com/riscv-non-isa/riscv-c-api-doc/blob/main/src/c-api.adoc #if defined(__riscv_zfhmin) diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt index 809decfbe5f08..5c9f622d44397 100644 --- a/libc/src/string/CMakeLists.txt +++ b/libc/src/string/CMakeLists.txt @@ -20,6 +20,7 @@ add_header_library( libc.hdr.stdint_proxy libc.src.__support.CPP.bitset libc.src.__support.CPP.type_traits + libc.src.__support.CPP.simd libc.src.__support.common ${string_config_options} ) diff --git a/libc/src/string/memory_utils/generic/inline_strlen.h b/libc/src/string/memory_utils/generic/inline_strlen.h new file mode 100644 index 0000000000000..111da35b85eeb --- /dev/null +++ b/libc/src/string/memory_utils/generic/inline_strlen.h @@ -0,0 +1,53 @@ +//===-- Strlen for generic SIMD types -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_INLINE_STRLEN_H +#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_INLINE_STRLEN_H + +#include "src/__support/CPP/simd.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE_DECL { +namespace internal { + +// Exploit the underlying integer representation to do a variable shift. +LIBC_INLINE constexpr cpp::simd_mask shift_mask(cpp::simd_mask m, + size_t shift) { + using bitmask_ty = cpp::internal::get_as_integer_type_t>; + bitmask_ty r = cpp::bit_cast(m) >> shift; + return cpp::bit_cast>(r); +} + +[[clang::no_sanitize("address")]] LIBC_INLINE size_t +string_length(const char *src) { + constexpr cpp::simd null_byte = cpp::splat('\0'); + + size_t alignment = alignof(cpp::simd); + const cpp::simd *aligned = reinterpret_cast *>( + __builtin_align_down(src, alignment)); + + cpp::simd chars = cpp::load_aligned>(aligned); + cpp::simd_mask mask = cpp::simd_cast(chars == null_byte); + size_t offset = src - reinterpret_cast(aligned); + if (cpp::any_of(shift_mask(mask, offset))) + return cpp::find_first_set(shift_mask(mask, offset)); + + for (;;) { + cpp::simd chars = cpp::load_aligned>(++aligned); + cpp::simd_mask mask = cpp::simd_cast(chars == null_byte); + if (cpp::any_of(mask)) + return (reinterpret_cast(aligned) - src) + + cpp::find_first_set(mask); + } +} +} // namespace internal + +namespace string_length_impl = internal; +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_INLINE_STRLEN_H diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h index ce461581b9d95..26e9adde0d66e 100644 --- a/libc/src/string/string_utils.h +++ b/libc/src/string/string_utils.h @@ -23,14 +23,16 @@ #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY #if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ) -#if defined(LIBC_TARGET_ARCH_IS_X86) +#if LIBC_HAS_VECTOR_TYPE +#include "src/string/memory_utils/generic/inline_strlen.h" +#elif defined(LIBC_TARGET_ARCH_IS_X86) #include "src/string/memory_utils/x86_64/inline_strlen.h" #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_NEON) #include "src/string/memory_utils/aarch64/inline_strlen.h" #else namespace string_length_impl = LIBC_NAMESPACE::wide_read; #endif -#endif +#endif // defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ) namespace LIBC_NAMESPACE_DECL { namespace internal { diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index bfda5385f012b..23ef774c18946 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -677,6 +677,18 @@ libc_support_library( ], ) +libc_support_library( + name = "__support_cpp_simd", + hdrs = ["src/__support/CPP/simd.h"], + deps = [ + ":__support_cpp_algorithm", + ":__support_cpp_bit", + ":__support_cpp_type_traits", + ":__support_macros_attributes", + ":hdr_stdint_proxy", + ], +) + libc_support_library( name = "__support_cpp_span", hdrs = ["src/__support/CPP/span.h"], @@ -4938,6 +4950,7 @@ libc_support_library( "src/string/memory_utils/arm/inline_memset.h", "src/string/memory_utils/generic/aligned_access.h", "src/string/memory_utils/generic/byte_per_byte.h", + "src/string/memory_utils/generic/inline_strlen.h", "src/string/memory_utils/inline_bcmp.h", "src/string/memory_utils/inline_bzero.h", "src/string/memory_utils/inline_memcmp.h", @@ -4964,6 +4977,7 @@ libc_support_library( ":__support_cpp_array", ":__support_cpp_bit", ":__support_cpp_cstddef", + ":__support_cpp_simd", ":__support_cpp_type_traits", ":__support_macros_attributes", ":__support_macros_optimization",