Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Create vector constants inline and not via ROS<byte> #2419

Merged
merged 4 commits into from
Mar 30, 2023
Merged

Create vector constants inline and not via ROS<byte> #2419

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
77ffeea
Create vector constants inline and not via ROS<byte>
gfoidl Mar 26, 2023
cdf6eed
Fixed build
gfoidl Mar 26, 2023
97ac149
Merge branch 'main' into vector-constants
gfoidl Mar 27, 2023
d617907
Merge branch 'main' into vector-constants
JimBobSquarePants Mar 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
113 changes: 55 additions & 58 deletions src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,33 +13,38 @@ internal static partial class SimdUtils
{
public static class HwIntrinsics
{
public static ReadOnlySpan<byte> PermuteMaskDeinterleave8x32 => new byte[] { 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)] // too much IL for JIT to inline, so give a hint
public static Vector256<int> PermuteMaskDeinterleave8x32() => Vector256.Create(0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0).AsInt32();

public static ReadOnlySpan<byte> PermuteMaskEvenOdd8x32 => new byte[] { 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskEvenOdd8x32() => Vector256.Create(0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32();

public static ReadOnlySpan<byte> PermuteMaskSwitchInnerDWords8x32 => new byte[] { 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskSwitchInnerDWords8x32() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0).AsUInt32();

private static ReadOnlySpan<byte> MoveFirst24BytesToSeparateLanes => new byte[] { 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<uint> MoveFirst24BytesToSeparateLanes() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32();

internal static ReadOnlySpan<byte> ExtractRgb => new byte[] { 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static Vector256<byte> ExtractRgb() => Vector256.Create(0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF);

private static ReadOnlySpan<byte> ShuffleMaskPad4Nx16 => new byte[] { 0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ShuffleMaskPad4Nx16() => Vector128.Create(0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);

private static ReadOnlySpan<byte> ShuffleMaskSlice4Nx16 => new byte[] { 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ShuffleMaskSlice4Nx16() => Vector128.Create(0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);

private static ReadOnlySpan<byte> ShuffleMaskShiftAlpha =>
new byte[]
{
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15,
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15
};
#pragma warning disable SA1003, SA1116, SA1117 // Parameters should be on same line or separate lines
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> ShuffleMaskShiftAlpha() => Vector256.Create((byte)
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15,
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15);

public static ReadOnlySpan<byte> PermuteMaskShiftAlpha8x32 =>
new byte[]
{
0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0,
5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0
};
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskShiftAlpha8x32() => Vector256.Create(
0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0,
5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
#pragma warning restore SA1003, SA1116, SA1117 // Parameters should be on same line or separate lines

/// <summary>
/// Shuffle single-precision (32-bit) floating-point elements in <paramref name="source"/>
Expand Down Expand Up @@ -189,7 +194,7 @@ public static class HwIntrinsics
{
if (Ssse3.IsSupported)
{
int remainder = source.Length % (Vector128<byte>.Count * 4);
int remainder = source.Length & (Vector128<byte>.Count * 4 - 1); // bit-hack for modulo

int sourceCount = source.Length - remainder;
int destCount = (int)((uint)sourceCount * 3 / 4);
Expand Down Expand Up @@ -221,7 +226,7 @@ public static class HwIntrinsics
ref Vector256<float> destBase =
ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(dest));

nint n = (nint)(uint)(dest.Length / Vector256<float>.Count);
nint n = (nint)((uint)dest.Length / (uint)Vector256<float>.Count);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've forgotten these in the other PR (I think because they're kept as nint due the operation in L231).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we should create a helper method for these? Feels very noisy seeing this before all our SIMD loops.

internal static partial class Numerics
{
    // Not sure about the name
    public static nint VectorCount<T>(Span<T> dest, int vectorSize) => (nint)((uint)dest.Length / (uint)vectorSize);
}

So then we can do:

nint n = Numerics.VectorCount(dest, Vector256<float>.Count);

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JimBobSquarePants meh or yeah?

Copy link
Contributor Author

@gfoidl gfoidl Mar 27, 2023 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the idea.
But it should be generic to account for different vector types w/o specifying the whole vector).

Maybe as extension to span?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But it should be generic to account for different vector types

I wish there was an IIntrinsicVector<T> interface with a static Count property but there isn't, and we are not on net7.0+ anyways :) So I'm just passing Vector256<float>.Count as a parameter in my recommendation, meaning that there is no way to check if the Vector256<T> and Span<T> element types are the same. Not ideal, but still better than writing out the verbose uint magic everywhere IMHO.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no way to check if the Vector256<T> and Span<T> element types are the same

They don't need to match. Reading e.g. Vector128<float> out of a ROS<byte> is pretty common.

So for the generic variant there could be two generic arguments. Here I don't like that the type of the span must be repeated. That should be infered (by the compiler).

In your I don't like that VectorXYZ<Type>.Count needs to be typed.

What do you think of this approach (IIRC it's only ROS/Span)?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you think of this approach (IIRC it's only ROS/Span)?

I like this option the most. We can have one overload for each span type.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Super.

I prefer doing this in a separete PR, as it doesn't match with this PR (this location is a left-over from the other PR) and there a quite a few places that need to be touched.
Fine?

Anyway I'll do this change later this afternoon (here or in a new PR).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New PR is fine for me!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--> #2422

nint m = Numerics.Modulo4(n);
nint u = n - m;

Expand Down Expand Up @@ -253,7 +258,7 @@ public static class HwIntrinsics
ref Vector128<float> destBase =
ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(dest));

nint n = (nint)(uint)dest.Length / Vector128<float>.Count;
nint n = (nint)((uint)dest.Length / (uint)Vector128<float>.Count);
nint m = Numerics.Modulo4(n);
nint u = n - m;

Expand Down Expand Up @@ -306,7 +311,7 @@ public static class HwIntrinsics
ref Vector256<byte> destBase =
ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(dest));

nint n = (nint)(uint)dest.Length / Vector256<byte>.Count;
nint n = (nint)((uint)dest.Length / (uint)Vector256<byte>.Count);
nint m = Numerics.Modulo4(n);
nint u = n - m;

Expand Down Expand Up @@ -342,7 +347,7 @@ public static class HwIntrinsics
ref Vector128<byte> destBase =
ref Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(dest));

nint n = (nint)(uint)dest.Length / Vector128<byte>.Count;
nint n = (nint)((uint)dest.Length / (uint)Vector128<byte>.Count);
nint m = Numerics.Modulo4(n);
nint u = n - m;

Expand Down Expand Up @@ -375,10 +380,8 @@ public static class HwIntrinsics
{
if (Ssse3.IsSupported)
{
ref byte vmaskBase = ref MemoryMarshal.GetReference(ShuffleMaskPad4Nx16);
Vector128<byte> vmask = Unsafe.As<byte, Vector128<byte>>(ref vmaskBase);
ref byte vmaskoBase = ref MemoryMarshal.GetReference(ShuffleMaskSlice4Nx16);
Vector128<byte> vmasko = Unsafe.As<byte, Vector128<byte>>(ref vmaskoBase);
Vector128<byte> vmask = ShuffleMaskPad4Nx16();
Vector128<byte> vmasko = ShuffleMaskSlice4Nx16();
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Expand Down Expand Up @@ -440,8 +443,7 @@ public static class HwIntrinsics
{
if (Ssse3.IsSupported)
{
ref byte vmaskBase = ref MemoryMarshal.GetReference(ShuffleMaskPad4Nx16);
Vector128<byte> vmask = Unsafe.As<byte, Vector128<byte>>(ref vmaskBase);
Vector128<byte> vmask = ShuffleMaskPad4Nx16();
Vector128<byte> vfill = Vector128.Create(0xff000000ff000000ul).AsByte();

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Expand Down Expand Up @@ -484,8 +486,7 @@ public static class HwIntrinsics
{
if (Ssse3.IsSupported)
{
ref byte vmaskoBase = ref MemoryMarshal.GetReference(ShuffleMaskSlice4Nx16);
Vector128<byte> vmasko = Unsafe.As<byte, Vector128<byte>>(ref vmaskoBase);
Vector128<byte> vmasko = ShuffleMaskSlice4Nx16();
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Expand Down Expand Up @@ -542,9 +543,9 @@ public static class HwIntrinsics
/// <returns>The <see cref="Vector256{T}"/>.</returns>
[MethodImpl(InliningOptions.AlwaysInline)]
public static Vector256<float> MultiplyAdd(
in Vector256<float> va,
in Vector256<float> vm0,
in Vector256<float> vm1)
Vector256<float> va,
Vector256<float> vm0,
Vector256<float> vm1)
{
if (Fma.IsSupported)
{
Expand All @@ -565,9 +566,9 @@ public static class HwIntrinsics
/// <returns>The <see cref="Vector256{T}"/>.</returns>
[MethodImpl(InliningOptions.ShortMethod)]
public static Vector256<float> MultiplySubtract(
in Vector256<float> vs,
in Vector256<float> vm0,
in Vector256<float> vm1)
Vector256<float> vs,
Vector256<float> vm0,
Vector256<float> vm1)
{
if (Fma.IsSupported)
{
Expand All @@ -587,9 +588,9 @@ public static class HwIntrinsics
/// <returns>The <see cref="Vector256{T}"/>.</returns>
[MethodImpl(InliningOptions.ShortMethod)]
public static Vector256<float> MultiplyAddNegated(
in Vector256<float> a,
in Vector256<float> b,
in Vector256<float> c)
Vector256<float> a,
Vector256<float> b,
Vector256<float> c)
{
if (Fma.IsSupported)
{
Expand Down Expand Up @@ -655,7 +656,7 @@ public static class HwIntrinsics
ref Vector256<float> destBase =
ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector256.Create(1 / (float)byte.MaxValue);
Vector256<float> scale = Vector256.Create(1 / (float)byte.MaxValue);

for (nuint i = 0; i < n; i++)
{
Expand Down Expand Up @@ -688,7 +689,7 @@ public static class HwIntrinsics
ref Vector128<float> destBase =
ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector128.Create(1 / (float)byte.MaxValue);
Vector128<float> scale = Vector128.Create(1 / (float)byte.MaxValue);
Vector128<byte> zero = Vector128<byte>.Zero;

for (nuint i = 0; i < n; i++)
Expand Down Expand Up @@ -790,9 +791,8 @@ public static class HwIntrinsics
ref Vector256<byte> destBase =
ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector256.Create((float)byte.MaxValue);
ref byte maskBase = ref MemoryMarshal.GetReference(PermuteMaskDeinterleave8x32);
Vector256<int> mask = Unsafe.As<byte, Vector256<int>>(ref maskBase);
Vector256<float> scale = Vector256.Create((float)byte.MaxValue);
Vector256<int> mask = PermuteMaskDeinterleave8x32();

for (nuint i = 0; i < n; i++)
{
Expand Down Expand Up @@ -829,7 +829,7 @@ public static class HwIntrinsics
ref Vector128<byte> destBase =
ref Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector128.Create((float)byte.MaxValue);
Vector128<float> scale = Vector128.Create((float)byte.MaxValue);

for (nuint i = 0; i < n; i++)
{
Expand Down Expand Up @@ -866,14 +866,12 @@ public static class HwIntrinsics

nuint count = (uint)redChannel.Length / (uint)Vector256<byte>.Count;

ref byte control1Bytes = ref MemoryMarshal.GetReference(PermuteMaskEvenOdd8x32);
Vector256<uint> control1 = Unsafe.As<byte, Vector256<uint>>(ref control1Bytes);
Vector256<uint> control1 = PermuteMaskEvenOdd8x32();

ref byte control2Bytes = ref MemoryMarshal.GetReference(PermuteMaskShiftAlpha8x32);
Vector256<uint> control2 = Unsafe.As<byte, Vector256<uint>>(ref control2Bytes);
var a = Vector256.Create((byte)255);
Vector256<uint> control2 = PermuteMaskShiftAlpha8x32();
Vector256<byte> a = Vector256.Create((byte)255);

Vector256<byte> shuffleAlpha = Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(ShuffleMaskShiftAlpha));
Vector256<byte> shuffleAlpha = ShuffleMaskShiftAlpha();

for (nuint i = 0; i < count; i++)
{
Expand Down Expand Up @@ -937,9 +935,8 @@ public static class HwIntrinsics
ref Vector256<byte> dBase = ref Unsafe.As<Rgba32, Vector256<byte>>(ref MemoryMarshal.GetReference(destination));

nuint count = (uint)redChannel.Length / (uint)Vector256<byte>.Count;
ref byte control1Bytes = ref MemoryMarshal.GetReference(PermuteMaskEvenOdd8x32);
Vector256<uint> control1 = Unsafe.As<byte, Vector256<uint>>(ref control1Bytes);
var a = Vector256.Create((byte)255);
Vector256<uint> control1 = PermuteMaskEvenOdd8x32();
Vector256<byte> a = Vector256.Create((byte)255);

for (nuint i = 0; i < count; i++)
{
Expand Down Expand Up @@ -988,8 +985,8 @@ public static class HwIntrinsics
ref Vector256<float> destGRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<float> destBRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(blueChannel));

Vector256<uint> extractToLanesMask = Unsafe.As<byte, Vector256<uint>>(ref MemoryMarshal.GetReference(MoveFirst24BytesToSeparateLanes));
Vector256<byte> extractRgbMask = Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(ExtractRgb));
Vector256<uint> extractToLanesMask = MoveFirst24BytesToSeparateLanes();
Vector256<byte> extractRgbMask = ExtractRgb();
Vector256<byte> rgb, rg, bx;
Vector256<float> r, g, b;

Expand Down
3 changes: 1 addition & 2 deletions tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,7 @@ public void Rgba32_Avx2_Float()

nuint count = (uint)this.Count / (uint)Vector256<float>.Count;

ref byte control = ref MemoryMarshal.GetReference(SimdUtils.HwIntrinsics.PermuteMaskEvenOdd8x32);
Vector256<int> vcontrol = Unsafe.As<byte, Vector256<int>>(ref control);
Vector256<int> vcontrol = SimdUtils.HwIntrinsics.PermuteMaskEvenOdd8x32().AsInt32();

var va = Vector256.Create(1F);

Expand Down