diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp index 7934b1eb91d..29a61936c49 100644 --- a/benchmarks/src/replace.cpp +++ b/benchmarks/src/replace.cpp @@ -35,7 +35,8 @@ void rc(benchmark::State& state) { } } -// replace() is vectorized for 4 and 8 bytes only. +BENCHMARK(r); +BENCHMARK(r); BENCHMARK(r); BENCHMARK(r); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 73e3b0d1380..9714d100cd5 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -121,6 +121,13 @@ __declspec(noalias) bool __stdcall __std_includes_less_8u( #endif // ^^^ _VECTORIZED_INCLUDES ^^^ #if _VECTORIZED_REPLACE +#if _VECTORIZED_REPLACE_1_2 +__declspec(noalias) void __stdcall __std_replace_1( + void* _First, void* _Last, uint8_t _Old_val, uint8_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_2( + void* _First, void* _Last, uint16_t _Old_val, uint16_t _New_val) noexcept; +#endif // ^^^ _VECTORIZED_REPLACE_1_2 ^^^ + // TRANSITION, DevCom-10610477 __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept; @@ -383,14 +390,25 @@ bool _Includes_vectorized( template __declspec(noalias) void _Replace_vectorized( _Ty* const _First, _Ty* const _Last, const _TVal1 _Old_val, const _TVal2 _New_val) noexcept { - if constexpr (sizeof(_Ty) == 4) { - ::__std_replace_4( - _First, _Last, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); - } else if constexpr (sizeof(_Ty) == 8) { - ::__std_replace_8( - _First, _Last, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); - } else { - static_assert(false, "unexpected size"); +#if _VECTORIZED_REPLACE_1_2 + if constexpr (sizeof(_Ty) == 1) { + ::__std_replace_1( + _First, _Last, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 2) { + ::__std_replace_2( + _First, _Last, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else +#endif // ^^^ _VECTORIZED_REPLACE_1_2 ^^^ + { + if constexpr (sizeof(_Ty) == 4) { + ::__std_replace_4( + _First, _Last, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 8) { + ::__std_replace_8( + _First, _Last, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else { + static_assert(false, "unexpected size"); + } } } #endif // ^^^ _VECTORIZED_REPLACE ^^^ @@ -491,10 +509,18 @@ _Ty* _Unique_copy_vectorized(const _Ty* const _First, const _Ty* const _Last, _T #endif // ^^^ _VECTORIZED_UNIQUE_COPY ^^^ #if _VECTORIZED_REPLACE +#if _VECTORIZED_REPLACE_1_2 +template +constexpr bool _Have_masked_op_for_iter = true; +#else // ^^^ _VECTORIZED_REPLACE_1_2 / !_VECTORIZED_REPLACE_1_2 vvv +template +constexpr bool _Have_masked_op_for_iter = sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size +#endif // ^^^ !_VECTORIZED_REPLACE_1_2 ^^^ + // Can we activate the vector algorithms for replace? template constexpr bool _Vector_alg_in_replace_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value - && sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size + && _Have_masked_op_for_iter<_Iter>; // Can we activate the vector algorithms for ranges::replace? template diff --git a/stl/inc/xutility b/stl/inc/xutility index a6537e5ad76..637aa0b3d99 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -58,17 +58,26 @@ _STL_DISABLE_CLANG_WARNINGS // In the STL's implementation, we inspect the following detection macros, // because various algorithms are vectorized for various architectures. -// _CALL_ALL_X64_VECTOR_ALGORITHMS_ON_ARM64EC is a macro to test ARM64EC fallbacks for x64 algorithms. - #if !_USE_STD_VECTOR_ALGORITHMS #define _VECTORIZED_FOR_X64_X86 0 +#define _VECTORIZED_FOR_X64_X86_ARM64 0 #define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 0 -#elif defined(_M_ARM64) || (defined(_M_ARM64EC) && !defined(_CALL_ALL_X64_VECTOR_ALGORITHMS_ON_ARM64EC)) +#define _VECTORIZED_FOR_ARM64 0 +#elif defined(_M_ARM64EC) && !defined(_CALL_ALL_X64_VECTOR_ALGORITHMS_ON_ARM64EC) // normal ARM64EC usage +#define _VECTORIZED_FOR_X64_X86 0 +#define _VECTORIZED_FOR_X64_X86_ARM64 0 +#define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 1 +#define _VECTORIZED_FOR_ARM64 0 +#elif defined(_M_ARM64) #define _VECTORIZED_FOR_X64_X86 0 +#define _VECTORIZED_FOR_X64_X86_ARM64 1 #define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 1 -#elif defined(_M_X64) || defined(_M_IX86) +#define _VECTORIZED_FOR_ARM64 1 +#elif defined(_M_X64) || defined(_M_IX86) // also when testing ARM64EC fallbacks for x64 algorithms #define _VECTORIZED_FOR_X64_X86 1 +#define _VECTORIZED_FOR_X64_X86_ARM64 1 #define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 1 +#define _VECTORIZED_FOR_ARM64 0 #else // ^^^ known architecture / unknown architecture vvv #error Unknown architecture #endif // ^^^ unknown architecture ^^^ @@ -89,7 +98,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_MISMATCH _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_REMOVE _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_REMOVE_COPY _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC -#define _VECTORIZED_REPLACE _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_REPLACE _VECTORIZED_FOR_X64_X86_ARM64 // not ARM64EC, which lacks SVE #define _VECTORIZED_REPLACE_COPY _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_REVERSE _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_REVERSE_COPY _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC @@ -104,6 +113,8 @@ _STL_DISABLE_CLANG_WARNINGS // as this does not improve performance over the scalar code. #define _VECTORIZED_MINMAX_ELEMENT_64BIT_INT _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_REPLACE_1_2 _VECTORIZED_FOR_ARM64 // not ARM64EC, which lacks SVE + #ifndef _USE_STD_VECTOR_FLOATING_ALGORITHMS #if _USE_STD_VECTOR_ALGORITHMS && !defined(_M_FP_EXCEPT) #define _USE_STD_VECTOR_FLOATING_ALGORITHMS 1 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 25dbc33e8d4..9a75e69620b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -11,17 +11,20 @@ #include #include +#if defined(_M_ARM64) // not ARM64EC, which lacks SVE +#include +#include + +extern "C" { +extern unsigned long long __arm64_xstate_features; // TRANSITION, __processor_features_0_63 will bake this in soon +} +#endif // ^^^ defined(_M_ARM64) ^^^ + #if defined(_M_ARM64) || defined(_M_ARM64EC) #include - -#include #else // ^^^ defined(_M_ARM64) || defined(_M_ARM64EC) / !defined(_M_ARM64) && !defined(_M_ARM64EC) vvv #include #include - -extern "C" { -extern int __isa_enabled; // TRANSITION, will declare this soon after 2026-06-09 -} #endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ namespace { @@ -54,43 +57,22 @@ namespace { } #endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ -#if defined(_M_ARM64) || defined(_M_ARM64EC) - bool _Use_FEAT_DotProd() noexcept { - return IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE); - } - - bool _Use_FEAT_I8MM() noexcept { - return IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE); - } - - bool _Use_FEAT_SHA3() noexcept { - return IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE); - } - +#if defined(_M_ARM64) // not ARM64EC, which lacks SVE bool _Use_FEAT_SVE() noexcept { - return IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE); - } + // defines the following constants: + constexpr int _Pf_arm_sve_instructions_available = 46; // PF_ARM_SVE_INSTRUCTIONS_AVAILABLE + constexpr int _Xstate_arm64_sve = 2; // XSTATE_ARM64_SVE + constexpr auto _Xstate_mask_arm64_sve = 1ull << _Xstate_arm64_sve; // XSTATE_MASK_ARM64_SVE - bool _Use_FEAT_SVE2() noexcept { - return IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE); - } + const bool _Sve_in_processor_features = + (__processor_features_0_63 & (1ull << _Pf_arm_sve_instructions_available)) != 0; - bool _Use_FEAT_SVE2p1() noexcept { - return IsProcessorFeaturePresent(PF_ARM_SVE2_1_INSTRUCTIONS_AVAILABLE); - } - - bool _Use_FEAT_SVE_SHA3() noexcept { - return IsProcessorFeaturePresent(PF_ARM_SVE_SHA3_INSTRUCTIONS_AVAILABLE); - } - - bool _Use_FEAT_AES() noexcept { - return IsProcessorFeaturePresent(PF_ARM_SVE_AES_INSTRUCTIONS_AVAILABLE); - } + // TRANSITION, __processor_features_0_63 will bake this in soon: + const bool _Sve_in_enabled_xstate_features = (__arm64_xstate_features & _Xstate_mask_arm64_sve) != 0; - bool _Use_FEAT_BitPerm() noexcept { - return IsProcessorFeaturePresent(PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE); + return _Sve_in_processor_features && _Sve_in_enabled_xstate_features; } -#endif // ^^^ defined(_M_ARM64) || defined(_M_ARM64EC) ^^^ +#endif // ^^^ defined(_M_ARM64) ^^^ size_t _Byte_length(const void* const _First, const void* const _Last) noexcept { return static_cast(_Last) - static_cast(_First); @@ -9608,6 +9590,121 @@ __declspec(noalias) size_t __stdcall __std_mismatch_8( namespace { namespace _Replacing { +#if defined(_M_ARM64) // not ARM64EC, which lacks SVE + struct _Traits_1_sve { + static svuint8_t _Load(const svbool_t _Pred, const void* const _Ptr) noexcept { + return svld1(_Pred, static_cast(_Ptr)); + } + + static svuint8_t _Set(const uint8_t _Val) noexcept { + return svdup_n_u8(_Val); + } + + static svbool_t _Cmp(const svbool_t _Pred, const svuint8_t _Lhs, const svuint8_t _Rhs) noexcept { + return svcmpeq(_Pred, _Lhs, _Rhs); + } + + static void _Store(const svbool_t _Pred, void* const _Ptr, const svuint8_t _Val) noexcept { + svst1(_Pred, static_cast(_Ptr), _Val); + } + }; + + struct _Traits_2_sve { + static svuint16_t _Load(const svbool_t _Pred, const void* const _Ptr) noexcept { + return svld1(_Pred, static_cast(_Ptr)); + } + + static svuint16_t _Set(const uint16_t _Val) noexcept { + return svdup_n_u16(_Val); + } + + static svbool_t _Cmp(const svbool_t _Pred, const svuint16_t _Lhs, const svuint16_t _Rhs) noexcept { + return svcmpeq(_Pred, _Lhs, _Rhs); + } + + static void _Store(const svbool_t _Pred, void* const _Ptr, const svuint16_t _Val) noexcept { + svst1(_Pred, static_cast(_Ptr), _Val); + } + }; + + struct _Traits_4_sve { + static svuint32_t _Load(const svbool_t _Pred, const void* const _Ptr) noexcept { + return svld1(_Pred, static_cast(_Ptr)); + } + + static svuint32_t _Set(const uint32_t _Val) noexcept { + return svdup_n_u32(_Val); + } + + static svbool_t _Cmp(const svbool_t _Pred, const svuint32_t _Lhs, const svuint32_t _Rhs) noexcept { + return svcmpeq(_Pred, _Lhs, _Rhs); + } + + static void _Store(const svbool_t _Pred, void* const _Ptr, const svuint32_t _Val) noexcept { + svst1(_Pred, static_cast(_Ptr), _Val); + } + }; + + struct _Traits_8_sve { + static svuint64_t _Load(const svbool_t _Pred, const void* const _Ptr) noexcept { + return svld1(_Pred, static_cast(_Ptr)); + } + + static svuint64_t _Set(const uint64_t _Val) noexcept { + return svdup_n_u64(_Val); + } + + static svbool_t _Cmp(const svbool_t _Pred, const svuint64_t _Lhs, const svuint64_t _Rhs) noexcept { + return svcmpeq(_Pred, _Lhs, _Rhs); + } + + static void _Store(const svbool_t _Pred, void* const _Ptr, const svuint64_t _Val) noexcept { + svst1(_Pred, static_cast(_Ptr), _Val); + } + }; + + template + __declspec(noalias) void __stdcall _Replace_impl( + void* _First, void* const _Last, const _Ty _Old_val, const _Ty _New_val) noexcept { + + if (_Use_FEAT_SVE()) { + // Arm Architecture Reference Manual for A-profile architecture, + // B1.4.2 "Configurable SVE vector lengths": + // "The architecturally defined SVL set is all powers of two from 128 to 2048 bits inclusive." + const size_t _Sve_vl = svcntb(); + const size_t _Size_bytes = _Byte_length(_First, _Last); + const size_t _Full_vl_bytes = _Size_bytes & ~size_t{_Sve_vl - 1}; + + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Full_vl_bytes); + + const auto _Comparand = _Traits::_Set(_Old_val); + const auto _Replacement = _Traits::_Set(_New_val); + + const auto _True = svptrue_b8(); + while (_First != _Stop_at) { + const auto _Data = _Traits::_Load(_True, _First); + const auto _Mask = _Traits::_Cmp(_True, _Data, _Comparand); + _Traits::_Store(_Mask, _First, _Replacement); + _Advance_bytes(_First, _Sve_vl); + } + + if (const size_t _Tail_length = _Size_bytes & size_t{_Sve_vl - 1}; _Tail_length != 0) { + const auto _Tail_mask = svwhilelt_b8(size_t{0}, _Tail_length); + const auto _Data = _Traits::_Load(_Tail_mask, _First); + const auto _Mask = _Traits::_Cmp(_Tail_mask, _Data, _Comparand); + _Traits::_Store(_Mask, _First, _Replacement); + } + } else { + for (auto _Cur = static_cast<_Ty*>(_First); _Cur != _Last; ++_Cur) { + if (*_Cur == _Old_val) { + *_Cur = _New_val; + } + } + } + } +#endif // ^^^ defined(_M_ARM64) ^^^ + #if defined(_M_ARM64) || defined(_M_ARM64EC) template __declspec(noalias) void __stdcall _Replace_copy_impl( @@ -9747,10 +9844,30 @@ namespace { extern "C" { -#ifndef _M_ARM64 +#if defined(_M_ARM64) // not ARM64EC, which lacks SVE +__declspec(noalias) void __stdcall __std_replace_1( + void* const _First, void* const _Last, const uint8_t _Old_val, const uint8_t _New_val) noexcept { + _Replacing::_Replace_impl<_Replacing::_Traits_1_sve>(_First, _Last, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_2( + void* const _First, void* const _Last, const uint16_t _Old_val, const uint16_t _New_val) noexcept { + _Replacing::_Replace_impl<_Replacing::_Traits_2_sve>(_First, _Last, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_4( + void* const _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { + _Replacing::_Replace_impl<_Replacing::_Traits_4_sve>(_First, _Last, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_8( + void* const _First, void* const _Last, const uint64_t _Old_val, const uint64_t _New_val) noexcept { + _Replacing::_Replace_impl<_Replacing::_Traits_8_sve>(_First, _Last, _Old_val, _New_val); +} +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { -#ifndef _M_ARM64EC +#if !defined(_M_ARM64EC) // not ARM64EC, which lacks AVX2 if (_Use_avx2()) { const __m256i _Comparand = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_Old_val)); const __m256i _Replacement = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_New_val)); @@ -9775,20 +9892,20 @@ __declspec(noalias) void __stdcall __std_replace_4( } _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } else + return; + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - { - for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { - if (*_Cur == _Old_val) { - *_Cur = _New_val; - } + + for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { + if (*_Cur == _Old_val) { + *_Cur = _New_val; } } } __declspec(noalias) void __stdcall __std_replace_8( void* _First, void* const _Last, const uint64_t _Old_val, const uint64_t _New_val) noexcept { -#ifndef _M_ARM64EC +#if !defined(_M_ARM64EC) // not ARM64EC, which lacks AVX2 if (_Use_avx2()) { #ifdef _WIN64 const __m256i _Comparand = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_Old_val)); @@ -9818,13 +9935,13 @@ __declspec(noalias) void __stdcall __std_replace_8( } _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } else + return; + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - { - for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { - if (*_Cur == _Old_val) { - *_Cur = _New_val; - } + + for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { + if (*_Cur == _Old_val) { + *_Cur = _New_val; } } } diff --git a/tests/std/include/test_vector_algorithms_support.hpp b/tests/std/include/test_vector_algorithms_support.hpp index c0aa608482b..979b8ce54ba 100644 --- a/tests/std/include/test_vector_algorithms_support.hpp +++ b/tests/std/include/test_vector_algorithms_support.hpp @@ -41,10 +41,6 @@ inline void initialize_randomness(std::mt19937_64& gen) { } #if (defined(_M_IX86) || defined(_M_X64)) && !defined(_M_CEE_PURE) -extern "C" { -extern int __isa_enabled; // TRANSITION, will declare this soon after 2026-06-09 -} - inline void disable_instructions(ISA_AVAILABILITY isa) { const unsigned long as_ulong = static_cast(isa); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index b46bc7babf6..7379a130fee 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -767,8 +767,13 @@ void test_case_replace_copy(const vector& input, vector& out_expected, vec template void test_replace(mt19937_64& gen) { - // replace() is vectorized for 4 and 8 bytes only. +#if defined(_M_ARM64) + // For ARM64, replace() is always vectorized. + constexpr bool replace_is_vectorized = true; +#else + // For x64/x86, replace() is vectorized for 4 and 8 bytes only. constexpr bool replace_is_vectorized = sizeof(T) >= 4; +#endif using TD = conditional_t; uniform_int_distribution dis(0, 9);