[not for review] Test SVE overhaul

benchmarks/src/replace.cpp

@@ -35,7 +35,8 @@ void rc(benchmark::State& state) {
     }
 }
 
-// replace() is vectorized for 4 and 8 bytes only.
+BENCHMARK(r<std::uint8_t>);
+BENCHMARK(r<std::uint16_t>);
 BENCHMARK(r<std::uint32_t>);
 BENCHMARK(r<std::uint64_t>);
 

stl/inc/algorithm

@@ -121,6 +121,13 @@ __declspec(noalias) bool __stdcall __std_includes_less_8u(
 #endif // ^^^ _VECTORIZED_INCLUDES ^^^
 
 #if _VECTORIZED_REPLACE
+#if _VECTORIZED_REPLACE_1_2
+__declspec(noalias) void __stdcall __std_replace_1(
+    void* _First, void* _Last, uint8_t _Old_val, uint8_t _New_val) noexcept;
+__declspec(noalias) void __stdcall __std_replace_2(
+    void* _First, void* _Last, uint16_t _Old_val, uint16_t _New_val) noexcept;
+#endif // ^^^ _VECTORIZED_REPLACE_1_2 ^^^
+
 // TRANSITION, DevCom-10610477
 __declspec(noalias) void __stdcall __std_replace_4(
     void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept;
@@ -383,14 +390,25 @@ bool _Includes_vectorized(
 template <class _Ty, class _TVal1, class _TVal2>
 __declspec(noalias) void _Replace_vectorized(
     _Ty* const _First, _Ty* const _Last, const _TVal1 _Old_val, const _TVal2 _New_val) noexcept {
-    if constexpr (sizeof(_Ty) == 4) {
-        ::__std_replace_4(
-            _First, _Last, _STD _Find_arg_cast<uint32_t>(_Old_val), _STD _Find_arg_cast<uint32_t>(_New_val));
-    } else if constexpr (sizeof(_Ty) == 8) {
-        ::__std_replace_8(
-            _First, _Last, _STD _Find_arg_cast<uint64_t>(_Old_val), _STD _Find_arg_cast<uint64_t>(_New_val));
-    } else {
-        static_assert(false, "unexpected size");
+#if _VECTORIZED_REPLACE_1_2
+    if constexpr (sizeof(_Ty) == 1) {
+        ::__std_replace_1(
+            _First, _Last, _STD _Find_arg_cast<uint8_t>(_Old_val), _STD _Find_arg_cast<uint8_t>(_New_val));
+    } else if constexpr (sizeof(_Ty) == 2) {
+        ::__std_replace_2(
+            _First, _Last, _STD _Find_arg_cast<uint16_t>(_Old_val), _STD _Find_arg_cast<uint16_t>(_New_val));
+    } else
+#endif // ^^^ _VECTORIZED_REPLACE_1_2 ^^^
+    {
+        if constexpr (sizeof(_Ty) == 4) {
+            ::__std_replace_4(
+                _First, _Last, _STD _Find_arg_cast<uint32_t>(_Old_val), _STD _Find_arg_cast<uint32_t>(_New_val));
+        } else if constexpr (sizeof(_Ty) == 8) {
+            ::__std_replace_8(
+                _First, _Last, _STD _Find_arg_cast<uint64_t>(_Old_val), _STD _Find_arg_cast<uint64_t>(_New_val));
+        } else {
+            static_assert(false, "unexpected size");
+        }
     }
 }
 #endif // ^^^ _VECTORIZED_REPLACE ^^^
@@ -491,10 +509,18 @@ _Ty* _Unique_copy_vectorized(const _Ty* const _First, const _Ty* const _Last, _T
 #endif // ^^^ _VECTORIZED_UNIQUE_COPY ^^^
 
 #if _VECTORIZED_REPLACE
+#if _VECTORIZED_REPLACE_1_2
+template <class _Iter>
+constexpr bool _Have_masked_op_for_iter = true;
+#else // ^^^ _VECTORIZED_REPLACE_1_2 / !_VECTORIZED_REPLACE_1_2 vvv
+template <class _Iter>
+constexpr bool _Have_masked_op_for_iter = sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size
+#endif // ^^^ !_VECTORIZED_REPLACE_1_2 ^^^
+
 // Can we activate the vector algorithms for replace?
 template <class _Iter, class _Ty1>
 constexpr bool _Vector_alg_in_replace_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value
-                                             && sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size
+                                             && _Have_masked_op_for_iter<_Iter>;
 
 // Can we activate the vector algorithms for ranges::replace?
 template <class _Iter, class _Ty1, class _Ty2>

stl/inc/xutility

@@ -58,17 +58,26 @@ _STL_DISABLE_CLANG_WARNINGS
 // In the STL's implementation, we inspect the following detection macros,
 // because various algorithms are vectorized for various architectures.
 
-// _CALL_ALL_X64_VECTOR_ALGORITHMS_ON_ARM64EC is a macro to test ARM64EC fallbacks for x64 algorithms.
-
 #if !_USE_STD_VECTOR_ALGORITHMS
 #define _VECTORIZED_FOR_X64_X86               0
+#define _VECTORIZED_FOR_X64_X86_ARM64         0
 #define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 0
-#elif defined(_M_ARM64) || (defined(_M_ARM64EC) && !defined(_CALL_ALL_X64_VECTOR_ALGORITHMS_ON_ARM64EC))
+#define _VECTORIZED_FOR_ARM64                 0
+#elif defined(_M_ARM64EC) && !defined(_CALL_ALL_X64_VECTOR_ALGORITHMS_ON_ARM64EC) // normal ARM64EC usage
+#define _VECTORIZED_FOR_X64_X86               0
+#define _VECTORIZED_FOR_X64_X86_ARM64         0
+#define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 1
+#define _VECTORIZED_FOR_ARM64                 0
+#elif defined(_M_ARM64)
 #define _VECTORIZED_FOR_X64_X86               0
+#define _VECTORIZED_FOR_X64_X86_ARM64         1
 #define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 1
-#elif defined(_M_X64) || defined(_M_IX86)
+#define _VECTORIZED_FOR_ARM64                 1
+#elif defined(_M_X64) || defined(_M_IX86) // also when testing ARM64EC fallbacks for x64 algorithms
 #define _VECTORIZED_FOR_X64_X86               1
+#define _VECTORIZED_FOR_X64_X86_ARM64         1
 #define _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC 1
+#define _VECTORIZED_FOR_ARM64                 0
 #else // ^^^ known architecture / unknown architecture vvv
 #error Unknown architecture
 #endif // ^^^ unknown architecture ^^^
@@ -89,7 +98,7 @@ _STL_DISABLE_CLANG_WARNINGS
 #define _VECTORIZED_MISMATCH           _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC
 #define _VECTORIZED_REMOVE             _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC
 #define _VECTORIZED_REMOVE_COPY        _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC
-#define _VECTORIZED_REPLACE            _VECTORIZED_FOR_X64_X86
+#define _VECTORIZED_REPLACE            _VECTORIZED_FOR_X64_X86_ARM64 // not ARM64EC, which lacks SVE
 #define _VECTORIZED_REPLACE_COPY       _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC
 #define _VECTORIZED_REVERSE            _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC
 #define _VECTORIZED_REVERSE_COPY       _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC
@@ -104,6 +113,8 @@ _STL_DISABLE_CLANG_WARNINGS
 // as this does not improve performance over the scalar code.
 #define _VECTORIZED_MINMAX_ELEMENT_64BIT_INT _VECTORIZED_FOR_X64_X86
 
+#define _VECTORIZED_REPLACE_1_2 _VECTORIZED_FOR_ARM64 // not ARM64EC, which lacks SVE
+
 #ifndef _USE_STD_VECTOR_FLOATING_ALGORITHMS
 #if _USE_STD_VECTOR_ALGORITHMS && !defined(_M_FP_EXCEPT)
 #define _USE_STD_VECTOR_FLOATING_ALGORITHMS 1