From 6e26a21f4fd5c448add3669d6c05c4a1566a2cbb Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Thu, 9 Jul 2020 14:50:22 -0700 Subject: [PATCH 01/11] AdvSimd support for System.Text.Unicode.Utf8Utility.TranscodeToUtf8 --- .../Text/Unicode/Utf8Utility.Transcoding.cs | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index ee9f789a40fab..b0a59713c7e89 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -4,8 +4,8 @@ using System.Buffers; using System.Diagnostics; using System.Numerics; -using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; #if SYSTEM_PRIVATE_CORELIB @@ -940,10 +940,8 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt uint inputCharsRemaining = (uint)(pFinalPosWhereCanReadDWordFromInputBuffer - pInputBuffer) + 2; uint minElementsRemaining = (uint)Math.Min(inputCharsRemaining, outputBytesRemaining); - if (Sse41.X64.IsSupported) + if ((AdvSimd.IsSupported || Sse41.X64.IsSupported) && BitConverter.IsLittleEndian) { - Debug.Assert(BitConverter.IsLittleEndian, "SSE41 requires little-endian."); - // Try reading and writing 8 elements per iteration. uint maxIters = minElementsRemaining / 8; ulong possibleNonAsciiQWord; @@ -957,9 +955,18 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt goto LoopTerminatedDueToNonAsciiDataInVectorLocal; } - // narrow and write + if (AdvSimd.IsSupported) + { + Vector64 lower = AdvSimd.ExtractNarrowingSaturateLower(utf16Data); + Vector128 source = AdvSimd.ExtractNarrowingSaturateUpper(lower, AdvSimd.LoadVector128((short*)pInputBuffer)); + AdvSimd.Store((ulong*)pOutputBuffer, source.AsUInt64()); + } + else + { + // narrow and write - Sse2.StoreScalar((ulong*)pOutputBuffer /* unaligned */, Sse2.PackUnsignedSaturate(utf16Data, utf16Data).AsUInt64()); + Sse2.StoreScalar((ulong*)pOutputBuffer /* unaligned */, Sse2.PackUnsignedSaturate(utf16Data, utf16Data).AsUInt64()); + } pInputBuffer += 8; pOutputBuffer += 8; @@ -978,7 +985,17 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt } utf16Data = Vector128.CreateScalarUnsafe(possibleNonAsciiQWord).AsInt16(); - Unsafe.WriteUnaligned(pOutputBuffer, Sse2.ConvertToUInt32(Sse2.PackUnsignedSaturate(utf16Data, utf16Data).AsUInt32())); + + if (AdvSimd.IsSupported) + { + Vector64 lower = AdvSimd.ExtractNarrowingSaturateUnsignedLower(utf16Data); + Vector128 source = AdvSimd.ExtractNarrowingSaturateUnsignedUpper(lower, AdvSimd.LoadVector128((short*)pInputBuffer + Vector128.Count)); + AdvSimd.StoreSelectedScalar((uint*)pOutputBuffer, source.AsUInt32(), 0); + } + else + { + Unsafe.WriteUnaligned(pOutputBuffer, Sse2.ConvertToUInt32(Sse2.PackUnsignedSaturate(utf16Data, utf16Data).AsUInt32())); + } pInputBuffer += 4; pOutputBuffer += 4; @@ -1000,7 +1017,16 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt if (Utf16Utility.AllCharsInUInt64AreAscii(possibleNonAsciiQWord)) // all chars in first QWORD are ASCII { - Unsafe.WriteUnaligned(pOutputBuffer, Sse2.ConvertToUInt32(Sse2.PackUnsignedSaturate(utf16Data, utf16Data).AsUInt32())); + if (AdvSimd.IsSupported) + { + Vector64 lower = AdvSimd.ExtractNarrowingSaturateUnsignedLower(utf16Data); + Vector128 source = AdvSimd.ExtractNarrowingSaturateUnsignedUpper(lower, AdvSimd.LoadVector128((short*)pOutputBuffer + Vector128.Count)); + Unsafe.WriteUnaligned(pOutputBuffer, source.AsUInt32().ToScalar()); + } + else + { + Unsafe.WriteUnaligned(pOutputBuffer, Sse2.ConvertToUInt32(Sse2.PackUnsignedSaturate(utf16Data, utf16Data).AsUInt32())); + } pInputBuffer += 4; pOutputBuffer += 4; outputBytesRemaining -= 4; From f596eaf0a152945b1d90f34c01485d8c3435979b Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Fri, 10 Jul 2020 13:41:42 -0700 Subject: [PATCH 02/11] Readd using to prevent build failure. Add AdvSimd equivalent operation to TestZ. --- .../Text/Unicode/Utf8Utility.Transcoding.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index b0a59713c7e89..7cfa53bc487d5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Diagnostics; using System.Numerics; +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; @@ -950,21 +951,26 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt for (i = 0; (uint)i < maxIters; i++) { utf16Data = Unsafe.ReadUnaligned>(pInputBuffer); - if (!Sse41.TestZ(utf16Data, nonAsciiUtf16DataMask)) - { - goto LoopTerminatedDueToNonAsciiDataInVectorLocal; - } if (AdvSimd.IsSupported) { + if (AdvSimd.CompareTest(utf16Data, nonAsciiUtf16DataMask).ToScalar() == 0) + { + goto LoopTerminatedDueToNonAsciiDataInVectorLocal; + } + Vector64 lower = AdvSimd.ExtractNarrowingSaturateLower(utf16Data); Vector128 source = AdvSimd.ExtractNarrowingSaturateUpper(lower, AdvSimd.LoadVector128((short*)pInputBuffer)); AdvSimd.Store((ulong*)pOutputBuffer, source.AsUInt64()); } else { - // narrow and write + if (!Sse41.TestZ(utf16Data, nonAsciiUtf16DataMask)) + { + goto LoopTerminatedDueToNonAsciiDataInVectorLocal; + } + // narrow and write Sse2.StoreScalar((ulong*)pOutputBuffer /* unaligned */, Sse2.PackUnsignedSaturate(utf16Data, utf16Data).AsUInt64()); } From f52aaf8dfb9710f9e61c0b21d592b55f010a7dba Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Tue, 14 Jul 2020 12:47:24 -0700 Subject: [PATCH 03/11] Inverted condition --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index 7cfa53bc487d5..ea598c54a2aa1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -954,7 +954,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt if (AdvSimd.IsSupported) { - if (AdvSimd.CompareTest(utf16Data, nonAsciiUtf16DataMask).ToScalar() == 0) + if (AdvSimd.CompareTest(utf16Data, nonAsciiUtf16DataMask).ToScalar() != 0) { goto LoopTerminatedDueToNonAsciiDataInVectorLocal; } From c967b878ea9431eca0c5872f564529e7dc48e6c5 Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Tue, 14 Jul 2020 17:25:51 -0700 Subject: [PATCH 04/11] Address IsSupported order, improve use ExtractNarrowingSaturated usage --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index ea598c54a2aa1..e2b0040e9d69e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -941,7 +941,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt uint inputCharsRemaining = (uint)(pFinalPosWhereCanReadDWordFromInputBuffer - pInputBuffer) + 2; uint minElementsRemaining = (uint)Math.Min(inputCharsRemaining, outputBytesRemaining); - if ((AdvSimd.IsSupported || Sse41.X64.IsSupported) && BitConverter.IsLittleEndian) + if (Sse41.X64.IsSupported || (AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian)) { // Try reading and writing 8 elements per iteration. uint maxIters = minElementsRemaining / 8; @@ -995,8 +995,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt if (AdvSimd.IsSupported) { Vector64 lower = AdvSimd.ExtractNarrowingSaturateUnsignedLower(utf16Data); - Vector128 source = AdvSimd.ExtractNarrowingSaturateUnsignedUpper(lower, AdvSimd.LoadVector128((short*)pInputBuffer + Vector128.Count)); - AdvSimd.StoreSelectedScalar((uint*)pOutputBuffer, source.AsUInt32(), 0); + AdvSimd.StoreSelectedScalar((uint*)pOutputBuffer, lower.AsUInt32(), 0); } else { @@ -1026,8 +1025,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt if (AdvSimd.IsSupported) { Vector64 lower = AdvSimd.ExtractNarrowingSaturateUnsignedLower(utf16Data); - Vector128 source = AdvSimd.ExtractNarrowingSaturateUnsignedUpper(lower, AdvSimd.LoadVector128((short*)pOutputBuffer + Vector128.Count)); - Unsafe.WriteUnaligned(pOutputBuffer, source.AsUInt32().ToScalar()); + AdvSimd.StoreSelectedScalar((uint*)pOutputBuffer, lower.AsUInt32(), 0); } else { From 328718b1b7d94fc9aaa2f05c6a5efcb593a669ac Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Tue, 14 Jul 2020 17:34:12 -0700 Subject: [PATCH 05/11] Rename source to result, second argument utf16Data --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index e2b0040e9d69e..e5c8477ba0b0a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -960,8 +960,8 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt } Vector64 lower = AdvSimd.ExtractNarrowingSaturateLower(utf16Data); - Vector128 source = AdvSimd.ExtractNarrowingSaturateUpper(lower, AdvSimd.LoadVector128((short*)pInputBuffer)); - AdvSimd.Store((ulong*)pOutputBuffer, source.AsUInt64()); + Vector128 result = AdvSimd.ExtractNarrowingSaturateUpper(lower, utf16Data); + AdvSimd.Store((ulong*)pOutputBuffer, result.AsUInt64()); } else { From 0024ad28d489fad8540821bc02b7e27c131323d2 Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Tue, 14 Jul 2020 18:31:08 -0700 Subject: [PATCH 06/11] Improve CompareTest --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index e5c8477ba0b0a..5cebfad864bb8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -954,7 +954,9 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt if (AdvSimd.IsSupported) { - if (AdvSimd.CompareTest(utf16Data, nonAsciiUtf16DataMask).ToScalar() != 0) + Vector128 isUtf16DataNonAscii = AdvSimd.CompareTest(utf16Data, nonAsciiUtf16DataMask); + bool hasNonAsciiDataInVector = AdvSimd.Arm64.MinAcross(isUtf16DataNonAscii).ToScalar() != 0; + if (hasNonAsciiDataInVector) { goto LoopTerminatedDueToNonAsciiDataInVectorLocal; } From 45d03d95749096c3de0455f151937cb2fb025d12 Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Tue, 14 Jul 2020 20:58:34 -0700 Subject: [PATCH 07/11] Add shims causing failures in Linux --- .../src/System/Runtime/Intrinsics/Intrinsics.Shims.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/System.Utf8String.Experimental/src/System/Runtime/Intrinsics/Intrinsics.Shims.cs b/src/libraries/System.Utf8String.Experimental/src/System/Runtime/Intrinsics/Intrinsics.Shims.cs index e5f9284769d6b..48d2859f3f06e 100644 --- a/src/libraries/System.Utf8String.Experimental/src/System/Runtime/Intrinsics/Intrinsics.Shims.cs +++ b/src/libraries/System.Utf8String.Experimental/src/System/Runtime/Intrinsics/Intrinsics.Shims.cs @@ -8,6 +8,7 @@ internal static class Vector64 public static Vector64 Create(ulong value) => throw new PlatformNotSupportedException(); public static Vector64 CreateScalar(uint value) => throw new PlatformNotSupportedException(); public static Vector64 AsByte(this Vector64 vector) where T : struct => throw new PlatformNotSupportedException(); + public static Vector64 AsUInt32(this Vector64 vector) where T : struct => throw new PlatformNotSupportedException(); } internal readonly struct Vector64 where T : struct From 191ec1fc90ea1692692f02b6e502d219a70bc99b Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Wed, 15 Jul 2020 13:14:17 -0700 Subject: [PATCH 08/11] Use unsigned version of ExtractNarrowingSaturate, avoid using MinAcross and use MaxPairwise instead --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index 5cebfad864bb8..52d7dfb3b3d76 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -955,15 +955,15 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt if (AdvSimd.IsSupported) { Vector128 isUtf16DataNonAscii = AdvSimd.CompareTest(utf16Data, nonAsciiUtf16DataMask); - bool hasNonAsciiDataInVector = AdvSimd.Arm64.MinAcross(isUtf16DataNonAscii).ToScalar() != 0; + bool hasNonAsciiDataInVector = AdvSimd.Arm64.MaxPairwise(isUtf16DataNonAscii, isUtf16DataNonAscii).AsUInt64().ToScalar() != 0; + if (hasNonAsciiDataInVector) { goto LoopTerminatedDueToNonAsciiDataInVectorLocal; } - Vector64 lower = AdvSimd.ExtractNarrowingSaturateLower(utf16Data); - Vector128 result = AdvSimd.ExtractNarrowingSaturateUpper(lower, utf16Data); - AdvSimd.Store((ulong*)pOutputBuffer, result.AsUInt64()); + Vector64 lower = AdvSimd.ExtractNarrowingSaturateUnsignedLower(utf16Data); + AdvSimd.Store(pOutputBuffer, lower); } else { From 1e5519c502f14c91427699ab606200c92a9530dd Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Thu, 16 Jul 2020 16:19:36 -0700 Subject: [PATCH 09/11] Missing support check for Sse2.X64 --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index 52d7dfb3b3d76..64e77c51dbcda 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -1014,7 +1014,15 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt LoopTerminatedDueToNonAsciiDataInVectorLocal: outputBytesRemaining -= 8 * i; - possibleNonAsciiQWord = Sse2.X64.ConvertToUInt64(utf16Data.AsUInt64()); + + if (Sse2.X64.IsSupported) + { + possibleNonAsciiQWord = Sse2.X64.ConvertToUInt64(utf16Data.AsUInt64()); + } + else + { + possibleNonAsciiQWord = utf16Data.AsUInt64().ToScalar(); + } // Temporarily set 'possibleNonAsciiQWord' to be the low 64 bits of the vector, // then check whether it's all-ASCII. If so, narrow and write to the destination From 86ef01f72b96a2c7315a14f09a32c0adc0a1fc59 Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Thu, 16 Jul 2020 16:59:08 -0700 Subject: [PATCH 10/11] Add missing case for AdvSimd --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index 64e77c51dbcda..acbab3ae34a0c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -883,7 +883,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt // is not enabled. Unsafe.SkipInit(out Vector128 nonAsciiUtf16DataMask); - if (Sse41.X64.IsSupported) + if (Sse41.X64.IsSupported || (AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian)) { nonAsciiUtf16DataMask = Vector128.Create(unchecked((short)0xFF80)); // mask of non-ASCII bits in a UTF-16 char } From 101c4f12bcef445c36180b10368cd1adf382f212 Mon Sep 17 00:00:00 2001 From: carlossanlop Date: Fri, 17 Jul 2020 17:20:52 -0700 Subject: [PATCH 11/11] Use MinPairwise for short --- .../src/System/Text/Unicode/Utf8Utility.Transcoding.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index acbab3ae34a0c..b0c1376611b9e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -955,7 +955,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt if (AdvSimd.IsSupported) { Vector128 isUtf16DataNonAscii = AdvSimd.CompareTest(utf16Data, nonAsciiUtf16DataMask); - bool hasNonAsciiDataInVector = AdvSimd.Arm64.MaxPairwise(isUtf16DataNonAscii, isUtf16DataNonAscii).AsUInt64().ToScalar() != 0; + bool hasNonAsciiDataInVector = AdvSimd.Arm64.MinPairwise(isUtf16DataNonAscii, isUtf16DataNonAscii).AsUInt64().ToScalar() != 0; if (hasNonAsciiDataInVector) {