diff --git a/libc/src/__support/CPP/simd.h b/libc/src/__support/CPP/simd.h index 3c7e65acc3c0a..d2a5b17fa4b9f 100644 --- a/libc/src/__support/CPP/simd.h +++ b/libc/src/__support/CPP/simd.h @@ -57,6 +57,29 @@ using simd = T [[clang::ext_vector_type(N)]]; template using simd_mask = simd>; +// Type trait helpers. +template +struct simd_size : cpp::integral_constant { +}; +template constexpr size_t simd_size_v = simd_size::value; + +template struct is_simd : cpp::integral_constant {}; +template +struct is_simd> : cpp::integral_constant {}; +template constexpr bool is_simd_v = is_simd::value; + +template +struct is_simd_mask : cpp::integral_constant {}; +template +struct is_simd_mask> : cpp::integral_constant {}; +template constexpr bool is_simd_mask_v = is_simd_mask::value; + +template struct simd_element_type; +template struct simd_element_type> { + using type = T; +}; +template +using simd_element_type_t = typename simd_element_type::type; namespace internal { template @@ -123,34 +146,14 @@ LIBC_INLINE constexpr static auto split(cpp::simd x) { return result; } -} // namespace internal - -// Type trait helpers. +// Helper trait template -struct simd_size : cpp::integral_constant { -}; -template constexpr size_t simd_size_v = simd_size::value; - -template struct is_simd : cpp::integral_constant {}; -template -struct is_simd> : cpp::integral_constant {}; -template constexpr bool is_simd_v = is_simd::value; - -template -struct is_simd_mask : cpp::integral_constant {}; -template -struct is_simd_mask> : cpp::integral_constant {}; -template constexpr bool is_simd_mask_v = is_simd_mask::value; +using enable_if_integral_t = cpp::enable_if_t, T>; -template struct simd_element_type; -template struct simd_element_type> { - using type = T; -}; template -using simd_element_type_t = typename simd_element_type::type; +using enable_if_simd_t = cpp::enable_if_t, bool>; -template -using enable_if_simd_t = cpp::enable_if_t, T>; +} // namespace internal // Casting. template @@ -159,29 +162,34 @@ LIBC_INLINE constexpr static simd simd_cast(simd v) { } // SIMD mask operations. -template LIBC_INLINE constexpr static bool all_of(simd m) { - return __builtin_reduce_and(m); +template = 0> +LIBC_INLINE constexpr static bool all_of(simd v) { + return __builtin_reduce_and(simd_cast(v)); } -template LIBC_INLINE constexpr static bool any_of(simd m) { - return __builtin_reduce_or(m); +template = 0> +LIBC_INLINE constexpr static bool any_of(simd v) { + return __builtin_reduce_or(simd_cast(v)); } -template LIBC_INLINE constexpr static bool none_of(simd m) { - return !any_of(m); +template = 0> +LIBC_INLINE constexpr static bool none_of(simd v) { + return !any_of(v); } -template LIBC_INLINE constexpr static bool some_of(simd m) { - return any_of(m) && !all_of(m); +template = 0> +LIBC_INLINE constexpr static bool some_of(simd v) { + return any_of(v) && !all_of(v); } -template LIBC_INLINE constexpr static int popcount(simd m) { - return __builtin_popcountg(m); +template = 0> +LIBC_INLINE constexpr static int popcount(simd v) { + return __builtin_popcountg(v); } -template -LIBC_INLINE constexpr static int find_first_set(simd m) { - return __builtin_ctzg(m); +template = 0> +LIBC_INLINE constexpr static int find_first_set(simd v) { + return __builtin_ctzg(simd_cast(v)); } -template -LIBC_INLINE constexpr static int find_last_set(simd m) { - constexpr size_t size = simd_size_v>; - return size - 1 - __builtin_clzg(m); +template = 0> +LIBC_INLINE constexpr static int find_last_set(simd v) { + constexpr size_t size = simd_size_v>; + return size - 1 - __builtin_clzg(simd_cast(v)); } // Elementwise operations. @@ -279,33 +287,32 @@ LIBC_INLINE constexpr static T hmax(simd v) { } // Accessor helpers. -template -LIBC_INLINE enable_if_simd_t load_unaligned(const void *ptr) { +template = 0> +LIBC_INLINE T load_unaligned(const void *ptr) { T tmp; __builtin_memcpy(&tmp, ptr, sizeof(T)); return tmp; } -template -LIBC_INLINE enable_if_simd_t load_aligned(const void *ptr) { +template = 0> +LIBC_INLINE T load_aligned(const void *ptr) { return load_unaligned(__builtin_assume_aligned(ptr, alignof(T))); } -template -LIBC_INLINE enable_if_simd_t store_unaligned(T v, void *ptr) { +template = 0> +LIBC_INLINE T store_unaligned(T v, void *ptr) { __builtin_memcpy(ptr, &v, sizeof(T)); } -template -LIBC_INLINE enable_if_simd_t store_aligned(T v, void *ptr) { +template = 0> +LIBC_INLINE T store_aligned(T v, void *ptr) { store_unaligned(v, __builtin_assume_aligned(ptr, alignof(T))); } -template -LIBC_INLINE enable_if_simd_t +template = 0> +LIBC_INLINE T masked_load(simd> m, void *ptr, T passthru = internal::poison>()) { return __builtin_masked_load(m, ptr, passthru); } -template -LIBC_INLINE enable_if_simd_t masked_store(simd> m, T v, - void *ptr) { +template = 0> +LIBC_INLINE T masked_store(simd> m, T v, void *ptr) { __builtin_masked_store( m, v, static_cast(__builtin_assume_aligned(ptr, alignof(T)))); } diff --git a/libc/src/__support/CPP/type_traits/is_unsigned.h b/libc/src/__support/CPP/type_traits/is_unsigned.h index 3ae6337ceb50a..b4267eedd19fc 100644 --- a/libc/src/__support/CPP/type_traits/is_unsigned.h +++ b/libc/src/__support/CPP/type_traits/is_unsigned.h @@ -16,6 +16,8 @@ #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" +#include + namespace LIBC_NAMESPACE_DECL { namespace cpp { @@ -46,6 +48,10 @@ template struct is_unsigned { LIBC_INLINE constexpr bool operator()() const { return is_unsigned::value; } }; #endif // LIBC_COMPILER_HAS_FIXED_POINT +#if LIBC_HAS_VECTOR_TYPE +template +struct is_unsigned : bool_constant {}; +#endif template LIBC_INLINE_VAR constexpr bool is_unsigned_v = is_unsigned::value; diff --git a/libc/src/string/memory_utils/generic/inline_strlen.h b/libc/src/string/memory_utils/generic/inline_strlen.h index 68fba2afb3a5c..5e553e301d4da 100644 --- a/libc/src/string/memory_utils/generic/inline_strlen.h +++ b/libc/src/string/memory_utils/generic/inline_strlen.h @@ -33,14 +33,14 @@ string_length(const char *src) { __builtin_align_down(src, alignment)); cpp::simd chars = cpp::load_aligned>(aligned); - cpp::simd_mask mask = cpp::simd_cast(chars == null_byte); + cpp::simd_mask mask = chars == null_byte; size_t offset = src - reinterpret_cast(aligned); if (cpp::any_of(shift_mask(mask, offset))) return cpp::find_first_set(shift_mask(mask, offset)); for (;;) { cpp::simd chars = cpp::load_aligned>(++aligned); - cpp::simd_mask mask = cpp::simd_cast(chars == null_byte); + cpp::simd_mask mask = chars == null_byte; if (cpp::any_of(mask)) return (reinterpret_cast(aligned) - src) + cpp::find_first_set(mask); diff --git a/libc/test/src/__support/CPP/simd_test.cpp b/libc/test/src/__support/CPP/simd_test.cpp index b4f5685e3b1d1..c8f34df8ab028 100644 --- a/libc/test/src/__support/CPP/simd_test.cpp +++ b/libc/test/src/__support/CPP/simd_test.cpp @@ -64,23 +64,25 @@ TEST(LlvmLibcSIMDTest, MaskOperations) { EXPECT_TRUE(cpp::any_of(mask)); EXPECT_FALSE(cpp::all_of(mask)); + EXPECT_FALSE(cpp::none_of(mask)); EXPECT_TRUE(cpp::some_of(mask)); EXPECT_EQ(cpp::find_first_set(mask), 0); EXPECT_EQ(cpp::find_last_set(mask), 2); + EXPECT_EQ(cpp::popcount(mask), 2); } TEST(LlvmLibcSIMDTest, SplitConcat) { cpp::simd v{1, 1, 2, 2, 3, 3, 4, 4}; auto [v1, v2, v3, v4] = cpp::split<2, 2, 2, 2>(v); - EXPECT_TRUE(cpp::all_of(cpp::simd_cast(v1 == 1))); - EXPECT_TRUE(cpp::all_of(cpp::simd_cast(v2 == 2))); - EXPECT_TRUE(cpp::all_of(cpp::simd_cast(v3 == 3))); - EXPECT_TRUE(cpp::all_of(cpp::simd_cast(v4 == 4))); + EXPECT_TRUE(cpp::all_of(v1 == 1)); + EXPECT_TRUE(cpp::all_of(v2 == 2)); + EXPECT_TRUE(cpp::all_of(v3 == 3)); + EXPECT_TRUE(cpp::all_of(v4 == 4)); cpp::simd m = cpp::concat(v1, v2, v3, v4); - EXPECT_TRUE(cpp::all_of(cpp::simd_cast(m == v))); + EXPECT_TRUE(cpp::all_of(m == v)); cpp::simd c(~0); cpp::simd n = cpp::concat(c, c, c, c, c, c, c, c); - EXPECT_TRUE(cpp::all_of(cpp::simd_cast(n == ~0))); + EXPECT_TRUE(cpp::all_of(n == ~0)); }