From d5901073661df95671e408070114c78abc2959b3 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Thu, 4 May 2023 01:47:36 +0200 Subject: [PATCH] Avoid bloom filter checks for IndexOfAnyExcept in ProbabilisticMap --- .../ProbabilisticCharSearchValues.cs | 22 +-- .../System/SearchValues/ProbabilisticMap.cs | 142 ++++++++++-------- 2 files changed, 83 insertions(+), 81 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticCharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticCharSearchValues.cs index a61f9b7bd210..fc7b01282162 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticCharSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticCharSearchValues.cs @@ -34,30 +34,16 @@ public ProbabilisticCharSearchValues(scoped ReadOnlySpan values) internal override bool ContainsCore(char value) => ProbabilisticMap.Contains(ref Unsafe.As(ref _map), _values, value); - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => - IndexOfAny(ref MemoryMarshal.GetReference(span), span.Length); + ProbabilisticMap.IndexOfAny(ref Unsafe.As(ref _map), ref MemoryMarshal.GetReference(span), span.Length, _values); - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => - IndexOfAny(ref MemoryMarshal.GetReference(span), span.Length); + ProbabilisticMap.IndexOfAnySimpleLoop(ref MemoryMarshal.GetReference(span), span.Length, _values); - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int LastIndexOfAny(ReadOnlySpan span) => - LastIndexOfAny(ref MemoryMarshal.GetReference(span), span.Length); + ProbabilisticMap.LastIndexOfAny(ref Unsafe.As(ref _map), ref MemoryMarshal.GetReference(span), span.Length, _values); - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - LastIndexOfAny(ref MemoryMarshal.GetReference(span), span.Length); - - [MethodImpl(MethodImplOptions.NoInlining)] - private int IndexOfAny(ref char searchSpace, int searchSpaceLength) - where TNegator : struct, IndexOfAnyAsciiSearcher.INegator => - ProbabilisticMap.IndexOfAny(ref Unsafe.As(ref _map), ref searchSpace, searchSpaceLength, _values); - - [MethodImpl(MethodImplOptions.NoInlining)] - private int LastIndexOfAny(ref char searchSpace, int searchSpaceLength) - where TNegator : struct, IndexOfAnyAsciiSearcher.INegator => - ProbabilisticMap.LastIndexOfAny(ref Unsafe.As(ref _map), ref searchSpace, searchSpaceLength, _values); + ProbabilisticMap.LastIndexOfAnySimpleLoop(ref MemoryMarshal.GetReference(span), span.Length, _values); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs index efd348aa99ae..099b162f85f1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs @@ -207,116 +207,96 @@ private static bool ShouldUseSimpleLoop(int searchSpaceLength, int valuesLength) || (searchSpaceLength < 20 && searchSpaceLength < (valuesLength >> 1)); } - public static int IndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) => - IndexOfAny>(ref searchSpace, searchSpaceLength, ref values, valuesLength); - - public static int IndexOfAnyExcept(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) => - IndexOfAny>(ref searchSpace, searchSpaceLength, ref values, valuesLength); - - public static int LastIndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) => - LastIndexOfAny>(ref searchSpace, searchSpaceLength, ref values, valuesLength); - - public static int LastIndexOfAnyExcept(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) => - LastIndexOfAny>(ref searchSpace, searchSpaceLength, ref values, valuesLength); - - private static int IndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) - where TNegator : struct, SpanHelpers.INegator + public static int IndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) { var valuesSpan = new ReadOnlySpan(ref values, valuesLength); // If the search space is relatively short compared to the needle, do a simple O(n * m) search. if (ShouldUseSimpleLoop(searchSpaceLength, valuesLength)) { - ref char searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength); - ref char cur = ref searchSpace; + return IndexOfAnySimpleLoop(ref searchSpace, searchSpaceLength, valuesSpan); + } - while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd)) - { - char c = cur; - if (TNegator.NegateIfNeeded(Contains(valuesSpan, c))) - { - return (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref cur) / sizeof(char)); - } + if (IndexOfAnyAsciiSearcher.TryIndexOfAny(ref searchSpace, searchSpaceLength, valuesSpan, out int index)) + { + return index; + } - cur = ref Unsafe.Add(ref cur, 1); - } + return ProbabilisticIndexOfAny(ref searchSpace, searchSpaceLength, ref values, valuesLength); + } - return -1; - } + public static int IndexOfAnyExcept(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) + { + var valuesSpan = new ReadOnlySpan(ref values, valuesLength); - if (typeof(TNegator) == typeof(SpanHelpers.DontNegate) - ? IndexOfAnyAsciiSearcher.TryIndexOfAny(ref searchSpace, searchSpaceLength, valuesSpan, out int index) - : IndexOfAnyAsciiSearcher.TryIndexOfAnyExcept(ref searchSpace, searchSpaceLength, valuesSpan, out index)) + if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && + !ShouldUseSimpleLoop(searchSpaceLength, valuesLength) && + IndexOfAnyAsciiSearcher.TryIndexOfAnyExcept(ref searchSpace, searchSpaceLength, valuesSpan, out int index)) { return index; } - return ProbabilisticIndexOfAny(ref searchSpace, searchSpaceLength, ref values, valuesLength); + return IndexOfAnySimpleLoop(ref searchSpace, searchSpaceLength, valuesSpan); } - private static int LastIndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) - where TNegator : struct, SpanHelpers.INegator + public static int LastIndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) { var valuesSpan = new ReadOnlySpan(ref values, valuesLength); // If the search space is relatively short compared to the needle, do a simple O(n * m) search. if (ShouldUseSimpleLoop(searchSpaceLength, valuesLength)) { - for (int i = searchSpaceLength - 1; i >= 0; i--) - { - char c = Unsafe.Add(ref searchSpace, i); - if (TNegator.NegateIfNeeded(Contains(valuesSpan, c))) - { - return i; - } - } + return LastIndexOfAnySimpleLoop(ref searchSpace, searchSpaceLength, valuesSpan); + } - return -1; + if (IndexOfAnyAsciiSearcher.TryLastIndexOfAny(ref searchSpace, searchSpaceLength, valuesSpan, out int index)) + { + return index; } - if (typeof(TNegator) == typeof(SpanHelpers.DontNegate) - ? IndexOfAnyAsciiSearcher.TryLastIndexOfAny(ref searchSpace, searchSpaceLength, valuesSpan, out int index) - : IndexOfAnyAsciiSearcher.TryLastIndexOfAnyExcept(ref searchSpace, searchSpaceLength, valuesSpan, out index)) + return ProbabilisticLastIndexOfAny(ref searchSpace, searchSpaceLength, ref values, valuesLength); + } + + public static int LastIndexOfAnyExcept(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) + { + var valuesSpan = new ReadOnlySpan(ref values, valuesLength); + + if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && + !ShouldUseSimpleLoop(searchSpaceLength, valuesLength) && + IndexOfAnyAsciiSearcher.TryLastIndexOfAnyExcept(ref searchSpace, searchSpaceLength, valuesSpan, out int index)) { return index; } - return ProbabilisticLastIndexOfAny(ref searchSpace, searchSpaceLength, ref values, valuesLength); + return LastIndexOfAnySimpleLoop(ref searchSpace, searchSpaceLength, valuesSpan); } [MethodImpl(MethodImplOptions.NoInlining)] - private static int ProbabilisticIndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) - where TNegator : struct, SpanHelpers.INegator + private static int ProbabilisticIndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) { var valuesSpan = new ReadOnlySpan(ref values, valuesLength); var map = new ProbabilisticMap(valuesSpan); ref uint charMap = ref Unsafe.As(ref map); - return typeof(TNegator) == typeof(SpanHelpers.DontNegate) - ? IndexOfAny(ref charMap, ref searchSpace, searchSpaceLength, valuesSpan) - : IndexOfAny(ref charMap, ref searchSpace, searchSpaceLength, valuesSpan); + return IndexOfAny(ref charMap, ref searchSpace, searchSpaceLength, valuesSpan); } [MethodImpl(MethodImplOptions.NoInlining)] - private static int ProbabilisticLastIndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) - where TNegator : struct, SpanHelpers.INegator + private static int ProbabilisticLastIndexOfAny(ref char searchSpace, int searchSpaceLength, ref char values, int valuesLength) { var valuesSpan = new ReadOnlySpan(ref values, valuesLength); var map = new ProbabilisticMap(valuesSpan); ref uint charMap = ref Unsafe.As(ref map); - return typeof(TNegator) == typeof(SpanHelpers.DontNegate) - ? LastIndexOfAny(ref charMap, ref searchSpace, searchSpaceLength, valuesSpan) - : LastIndexOfAny(ref charMap, ref searchSpace, searchSpaceLength, valuesSpan); + return LastIndexOfAny(ref charMap, ref searchSpace, searchSpaceLength, valuesSpan); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfAny(ref uint charMap, ref char searchSpace, int searchSpaceLength, ReadOnlySpan values) - where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + internal static int IndexOfAny(ref uint charMap, ref char searchSpace, int searchSpaceLength, ReadOnlySpan values) { - if ((Sse41.IsSupported || AdvSimd.Arm64.IsSupported) && typeof(TNegator) == typeof(IndexOfAnyAsciiSearcher.DontNegate) && searchSpaceLength >= 16) + if ((Sse41.IsSupported || AdvSimd.Arm64.IsSupported) && searchSpaceLength >= 16) { return IndexOfAnyVectorized(ref charMap, ref searchSpace, searchSpaceLength, values); } @@ -327,7 +307,7 @@ internal static int IndexOfAny(ref uint charMap, ref char searchSpace, while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd)) { int ch = cur; - if (TNegator.NegateIfNeeded(Contains(ref charMap, values, ch))) + if (Contains(ref charMap, values, ch)) { return (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref cur) / sizeof(char)); } @@ -339,13 +319,12 @@ internal static int IndexOfAny(ref uint charMap, ref char searchSpace, } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int LastIndexOfAny(ref uint charMap, ref char searchSpace, int searchSpaceLength, ReadOnlySpan values) - where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + internal static int LastIndexOfAny(ref uint charMap, ref char searchSpace, int searchSpaceLength, ReadOnlySpan values) { for (int i = searchSpaceLength - 1; i >= 0; i--) { int ch = Unsafe.Add(ref searchSpace, i); - if (TNegator.NegateIfNeeded(Contains(ref charMap, values, ch))) + if (Contains(ref charMap, values, ch)) { return i; } @@ -461,5 +440,42 @@ private static int IndexOfAnyVectorized(ref uint charMap, ref char searchSpace, return -1; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int IndexOfAnySimpleLoop(ref char searchSpace, int searchSpaceLength, ReadOnlySpan values) + where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + { + ref char searchSpaceEnd = ref Unsafe.Add(ref searchSpace, searchSpaceLength); + ref char cur = ref searchSpace; + + while (!Unsafe.AreSame(ref cur, ref searchSpaceEnd)) + { + char c = cur; + if (TNegator.NegateIfNeeded(Contains(values, c))) + { + return (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref cur) / sizeof(char)); + } + + cur = ref Unsafe.Add(ref cur, 1); + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int LastIndexOfAnySimpleLoop(ref char searchSpace, int searchSpaceLength, ReadOnlySpan values) + where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + { + for (int i = searchSpaceLength - 1; i >= 0; i--) + { + char c = Unsafe.Add(ref searchSpace, i); + if (TNegator.NegateIfNeeded(Contains(values, c))) + { + return i; + } + } + + return -1; + } } }