Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a dedicated Ascii.IsValid path #84881

Merged
merged 6 commits into from Apr 20, 2023
Merged

Add a dedicated Ascii.IsValid path #84881

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bc470fd
Add dedicated Ascii.IsValid path
MihaZupan Apr 15, 2023
9a97e63
Remove wrong assert
MihaZupan Apr 15, 2023
5570632
Use for loop instead of chasing end byref
MihaZupan Apr 19, 2023
d3f100b
Add an extra assert
MihaZupan Apr 19, 2023
fbc1903
Use nuint in loop for [0, 3] elements
MihaZupan Apr 19, 2023
59211c2
Remove leftover byref increment
MihaZupan Apr 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
28 changes: 2 additions & 26 deletions src/libraries/System.Private.CoreLib/src/System/Text/Ascii.CaseConversion.cs
View file
Open in desktop
Expand Up @@ -238,7 +238,7 @@ public static OperationStatus ToUpperInPlace(Span<char> value, out int charsWrit
// Unaligned read and check for non-ASCII data.

Vector128<TFrom> srcVector = Vector128.LoadUnsafe(ref *pSrc);
if (VectorContainsAnyNonAsciiData(srcVector))
if (VectorContainsNonAsciiChar(srcVector))
{
goto Drain64;
}
Expand Down Expand Up @@ -291,7 +291,7 @@ public static OperationStatus ToUpperInPlace(Span<char> value, out int charsWrit
// Unaligned read & check for non-ASCII data.

srcVector = Vector128.LoadUnsafe(ref *pSrc, i);
if (VectorContainsAnyNonAsciiData(srcVector))
if (VectorContainsNonAsciiChar(srcVector))
{
goto Drain64;
}
Expand Down Expand Up @@ -463,30 +463,6 @@ public static OperationStatus ToUpperInPlace(Span<char> value, out int charsWrit
return i;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe bool VectorContainsAnyNonAsciiData<T>(Vector128<T> vector)
where T : unmanaged
{
if (sizeof(T) == 1)
{
if (vector.ExtractMostSignificantBits() != 0) { return true; }
}
else if (sizeof(T) == 2)
{
if (VectorContainsNonAsciiChar(vector.AsUInt16()))
{
return true;
}
}
else
{
Debug.Fail("Unknown types provided.");
throw new NotSupportedException();
}

return false;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void Widen8To16AndAndWriteTo(Vector128<byte> narrowVector, char* pDest, nuint destOffset)
{
Expand Down
57 changes: 57 additions & 0 deletions src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs
View file
Open in desktop
Expand Up @@ -41,6 +41,17 @@ private static bool AllCharsInUInt64AreAscii(ulong value)
return (value & ~0x007F007F_007F007Ful) == 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AllCharsInUInt64AreAscii<T>(ulong value)
where T : unmanaged
{
Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));

return typeof(T) == typeof(byte)
? AllBytesInUInt64AreAscii(value)
: AllCharsInUInt64AreAscii(value);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetIndexOfFirstNonAsciiByteInLane_AdvSimd(Vector128<byte> value, Vector128<byte> bitmask)
{
Expand Down Expand Up @@ -1432,6 +1443,52 @@ private static bool VectorContainsNonAsciiChar(Vector128<ushort> utf16Vector)
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool VectorContainsNonAsciiChar<T>(Vector128<T> vector)
where T : unmanaged
{
Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));

return typeof(T) == typeof(byte)
? VectorContainsNonAsciiChar(vector.AsByte())
: VectorContainsNonAsciiChar(vector.AsUInt16());
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AllCharsInVectorAreAscii<T>(Vector128<T> vector)
where T : unmanaged
{
Debug.Assert(Avx.IsSupported);
Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));

if (typeof(T) == typeof(byte))
{
return
Sse41.IsSupported ? Sse41.TestZ(vector.AsByte(), Vector128.Create((byte)0x80)) :
AdvSimd.Arm64.IsSupported ? AllBytesInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsByte(), vector.AsByte()).AsUInt64().ToScalar()) :
vector.AsByte().ExtractMostSignificantBits() == 0;
}
else
{
return
Sse41.IsSupported ? Sse41.TestZ(vector.AsInt16(), Vector128.Create((short)-128)) :
AdvSimd.Arm64.IsSupported ? AllCharsInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsUInt16(), vector.AsUInt16()).AsUInt64().ToScalar()) :
(vector.AsUInt16() & Vector128.Create((ushort)(ushort.MaxValue - 127))) == Vector128<ushort>.Zero;
EgorBo marked this conversation as resolved.
Show resolved Hide resolved
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AllCharsInVectorAreAscii<T>(Vector256<T> vector)
where T : unmanaged
{
Debug.Assert(Avx.IsSupported);
Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));

return typeof(T) == typeof(byte)
? Avx.TestZ(vector.AsByte(), Vector256.Create((byte)0x80))
: Avx.TestZ(vector.AsInt16(), Vector256.Create((short)-128));
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ExtractAsciiVector(Vector128<ushort> vectorFirst, Vector128<ushort> vectorSecond)
{
Expand Down
225 changes: 193 additions & 32 deletions src/libraries/System.Private.CoreLib/src/System/Text/Ascii.cs
View file
Open in desktop
Expand Up @@ -2,7 +2,9 @@
// The .NET Foundation licenses this file to you under the MIT license.

using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;

namespace System.Text
{
Expand All @@ -14,56 +16,215 @@ public static partial class Ascii
/// <param name="value">The value to inspect.</param>
/// <returns>True if <paramref name="value"/> contains only ASCII bytes or is
/// empty; False otherwise.</returns>
public static unsafe bool IsValid(ReadOnlySpan<byte> value)
{
if (value.IsEmpty)
{
return true;
}

nuint bufferLength = (uint)value.Length;
fixed (byte* pBuffer = &MemoryMarshal.GetReference(value))
{
nuint idxOfFirstNonAsciiElement = GetIndexOfFirstNonAsciiByte(pBuffer, bufferLength);
Debug.Assert(idxOfFirstNonAsciiElement <= bufferLength);
return idxOfFirstNonAsciiElement == bufferLength;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsValid(ReadOnlySpan<byte> value) =>
IsValidCore(ref MemoryMarshal.GetReference(value), value.Length);

/// <summary>
/// Determines whether the provided value contains only ASCII chars.
/// </summary>
/// <param name="value">The value to inspect.</param>
/// <returns>True if <paramref name="value"/> contains only ASCII chars or is
/// empty; False otherwise.</returns>
public static unsafe bool IsValid(ReadOnlySpan<char> value)
{
if (value.IsEmpty)
{
return true;
}

nuint bufferLength = (uint)value.Length;
fixed (char* pBuffer = &MemoryMarshal.GetReference(value))
{
nuint idxOfFirstNonAsciiElement = GetIndexOfFirstNonAsciiChar(pBuffer, bufferLength);
Debug.Assert(idxOfFirstNonAsciiElement <= bufferLength);
return idxOfFirstNonAsciiElement == bufferLength;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsValid(ReadOnlySpan<char> value) =>
IsValidCore(ref Unsafe.As<char, ushort>(ref MemoryMarshal.GetReference(value)), value.Length);

/// <summary>
/// Determines whether the provided value is ASCII byte.
/// </summary>
/// <param name="value">The value to inspect.</param>
/// <returns>True if <paramref name="value"/> is ASCII, False otherwise.</returns>
public static unsafe bool IsValid(byte value) => value <= 127;
public static bool IsValid(byte value) => value <= 127;

/// <summary>
/// Determines whether the provided value is ASCII char.
/// </summary>
/// <param name="value">The value to inspect.</param>
/// <returns>True if <paramref name="value"/> is ASCII, False otherwise.</returns>
public static unsafe bool IsValid(char value) => value <= 127;
public static bool IsValid(char value) => value <= 127;

private static unsafe bool IsValidCore<T>(ref T searchSpace, int length) where T : unmanaged
{
Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));

ref T searchSpaceEnd = ref Unsafe.Add(ref searchSpace, length);

if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
{
int elementsPerUlong = sizeof(ulong) / sizeof(T);

if (length < elementsPerUlong)
{
if (typeof(T) == typeof(byte) && length >= sizeof(uint))
{
// Process byte inputs with lengths [4, 7]
return AllBytesInUInt32AreAscii(
Unsafe.ReadUnaligned<uint>(ref Unsafe.As<T, byte>(ref searchSpace)) |
Unsafe.ReadUnaligned<uint>(ref Unsafe.As<T, byte>(ref Unsafe.Subtract(ref searchSpaceEnd, sizeof(uint)))));
}

// Process inputs with lengths [0, 3]
while (Unsafe.IsAddressLessThan(ref searchSpace, ref searchSpaceEnd))
{
if (typeof(T) == typeof(byte)
? (Unsafe.BitCast<T, byte>(searchSpace) > 127)
: (Unsafe.BitCast<T, char>(searchSpace) > 127))
{
return false;
}

searchSpace = ref Unsafe.Add(ref searchSpace, 1);
}

return true;
}

// If vectorization isn't supported, process 16 bytes at a time.
if (!Vector128.IsHardwareAccelerated && length > 2 * elementsPerUlong)
{
ref T finalStart = ref Unsafe.Subtract(ref searchSpaceEnd, 2 * elementsPerUlong);

do
{
if (!AllCharsInUInt64AreAscii<T>(
Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<T, byte>(ref searchSpace)) |
Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref Unsafe.As<T, byte>(ref searchSpace), sizeof(ulong)))))
{
return false;
}

searchSpace = ref Unsafe.Add(ref searchSpace, 2 * elementsPerUlong);
}
while (Unsafe.IsAddressLessThan(ref searchSpace, ref finalStart));

searchSpace = ref finalStart;
}

// Process the last [8, 16] bytes.
return AllCharsInUInt64AreAscii<T>(
Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<T, byte>(ref searchSpace)) |
Unsafe.ReadUnaligned<ulong>(ref Unsafe.Subtract(ref Unsafe.As<T, byte>(ref searchSpaceEnd), sizeof(ulong))));
}

// Process inputs with lengths [16, 32] bytes.
if (length <= 2 * Vector128<T>.Count)
{
return AllCharsInVectorAreAscii(
Vector128.LoadUnsafe(ref searchSpace) |
Vector128.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, (nuint)Vector128<T>.Count)));
}

if (Vector256.IsHardwareAccelerated)
{
// Process inputs with lengths [33, 64] bytes.
if (length <= 2 * Vector256<T>.Count)
{
return AllCharsInVectorAreAscii(
Vector256.LoadUnsafe(ref searchSpace) |
Vector256.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, (nuint)Vector256<T>.Count)));
}

// Process long inputs 128 bytes at a time.
if (length > 4 * Vector256<T>.Count)
{
// Process the first 128 bytes.
if (!AllCharsInVectorAreAscii(
Vector256.LoadUnsafe(ref searchSpace) |
Vector256.LoadUnsafe(ref searchSpace, (nuint)Vector256<T>.Count) |
Vector256.LoadUnsafe(ref searchSpace, 2 * (nuint)Vector256<T>.Count) |
Vector256.LoadUnsafe(ref searchSpace, 3 * (nuint)Vector256<T>.Count)))
stephentoub marked this conversation as resolved.
Show resolved Hide resolved
{
return false;
}

searchSpace = ref Unsafe.Add(ref searchSpace, 4 * Vector256<T>.Count);

// Try to opportunistically align the reads below. The input isn't pinned, so the GC
// is free to move the references. We're therefore assuming that reads may still be unaligned.
// They may also be unaligned if the input chars aren't 2-byte aligned.
nuint misalignedElements = ((nuint)Unsafe.AsPointer(ref searchSpace) & (nuint)(Vector256<byte>.Count - 1)) / (nuint)sizeof(T);
searchSpace = ref Unsafe.Subtract(ref searchSpace, misalignedElements);

ref T finalStart = ref Unsafe.Subtract(ref searchSpaceEnd, 4 * Vector256<T>.Count);

while (Unsafe.IsAddressLessThan(ref searchSpace, ref finalStart))
{
if (!AllCharsInVectorAreAscii(
Vector256.LoadUnsafe(ref searchSpace) |
Vector256.LoadUnsafe(ref searchSpace, (nuint)Vector256<T>.Count) |
Vector256.LoadUnsafe(ref searchSpace, 2 * (nuint)Vector256<T>.Count) |
Vector256.LoadUnsafe(ref searchSpace, 3 * (nuint)Vector256<T>.Count)))
{
return false;
}

searchSpace = ref Unsafe.Add(ref searchSpace, 4 * Vector256<T>.Count);
}

searchSpace = ref finalStart;
}

// Process the last [1, 128] bytes.
// The search space has at least 2 * Vector256 bytes available to read.
// We process the first 2 and last 2 vectors, which may overlap.
return AllCharsInVectorAreAscii(
Vector256.LoadUnsafe(ref searchSpace) |
Vector256.LoadUnsafe(ref searchSpace, (nuint)Vector256<T>.Count) |
Vector256.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, 2 * (nuint)Vector256<T>.Count)) |
Vector256.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, (nuint)Vector256<T>.Count)));
}
else
{
// Process long inputs 64 bytes at a time.
if (length > 4 * Vector128<T>.Count)
{
// Process the first 64 bytes.
if (!AllCharsInVectorAreAscii(
Vector128.LoadUnsafe(ref searchSpace) |
Vector128.LoadUnsafe(ref searchSpace, (nuint)Vector128<T>.Count) |
Vector128.LoadUnsafe(ref searchSpace, 2 * (nuint)Vector128<T>.Count) |
Vector128.LoadUnsafe(ref searchSpace, 3 * (nuint)Vector128<T>.Count)))
{
return false;
}

searchSpace = ref Unsafe.Add(ref searchSpace, 4 * Vector128<T>.Count);

// Try to opportunistically align the reads below. The input isn't pinned, so the GC
// is free to move the references. We're therefore assuming that reads may still be unaligned.
// They may also be unaligned if the input chars aren't 2-byte aligned.
nuint misalignedElements = ((nuint)Unsafe.AsPointer(ref searchSpace) & (nuint)(Vector128<byte>.Count - 1)) / (nuint)sizeof(T);
searchSpace = ref Unsafe.Subtract(ref searchSpace, misalignedElements);

ref T finalStart = ref Unsafe.Subtract(ref searchSpaceEnd, 4 * Vector128<T>.Count);

while (Unsafe.IsAddressLessThan(ref searchSpace, ref finalStart))
MihaZupan marked this conversation as resolved.
Show resolved Hide resolved
{
if (!AllCharsInVectorAreAscii(
Vector128.LoadUnsafe(ref searchSpace) |
Vector128.LoadUnsafe(ref searchSpace, (nuint)Vector128<T>.Count) |
Vector128.LoadUnsafe(ref searchSpace, 2 * (nuint)Vector128<T>.Count) |
Vector128.LoadUnsafe(ref searchSpace, 3 * (nuint)Vector128<T>.Count)))
{
return false;
}

searchSpace = ref Unsafe.Add(ref searchSpace, 4 * Vector128<T>.Count);
}

searchSpace = ref finalStart;
}

// Process the last [1, 64] bytes.
// The search space has at least 2 * Vector128 bytes available to read.
// We process the first 2 and last 2 vectors, which may overlap.
return AllCharsInVectorAreAscii(
Vector128.LoadUnsafe(ref searchSpace) |
Vector128.LoadUnsafe(ref searchSpace, (nuint)Vector128<T>.Count) |
Vector128.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, 2 * (nuint)Vector128<T>.Count)) |
Vector128.LoadUnsafe(ref Unsafe.Subtract(ref searchSpaceEnd, (nuint)Vector128<T>.Count)));
}
}
}
}