diff --git a/ColorChord.NET/NoteFinder/ShinNoteFinder.cs b/ColorChord.NET/NoteFinder/ShinNoteFinder.cs index 3870a9d..3141b23 100644 --- a/ColorChord.NET/NoteFinder/ShinNoteFinder.cs +++ b/ColorChord.NET/NoteFinder/ShinNoteFinder.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using System.Threading; namespace ColorChord.NET.NoteFinder; @@ -24,6 +25,8 @@ public class ShinNoteFinder : NoteFinderCommon, ITimingSource private int[] P_PersistentNoteIDs; public override ReadOnlySpan PersistentNoteIDs => this.P_PersistentNoteIDs; + public byte[] PeakBits, WidebandBits; + private static Thread? ProcessThread; private static bool KeepGoing = true; @@ -42,6 +45,8 @@ public ShinNoteFinder(string name, Dictionary config) P_PersistentNoteIDs = new int[NOTE_QTY]; SetupBuffers(); ShinNoteFinderDFT.Reconfigure(); + PeakBits = new byte[ShinNoteFinderDFT.BinCount / 8]; // TODO: Assumes BinsPerOctave is divisible by 8, not true for 12 + WidebandBits = new byte[ShinNoteFinderDFT.BinCount / 8]; } public override int NoteCount => NOTE_QTY; @@ -94,9 +99,77 @@ private static void Cycle() public override void UpdateOutputs() { ShinNoteFinderDFT.CalculateOutput(); - for (int i = 0; i < AllBinValues.Length; i++) + + float[] RawBinValuesPadded = ShinNoteFinderDFT.AllBinValues; + + Debug.Assert(RawBinValuesPadded[0] == 0F, "AllBinValues left boundary value was changed, it should always be 0."); + Debug.Assert(RawBinValuesPadded[^1] == 0F, "AllBinValues right boundary value was changed, it should always be 0."); + + byte[] PeakBitsPacked = this.PeakBits; + byte[] WidebandBitsPacked = this.WidebandBits; + + for (int i = 0; i < WidebandBitsPacked.Length; i++) { WidebandBitsPacked[i] = 0; } + + for (int Step = 0; Step <= RawBinValuesPadded.Length - 10; Step += 8) + { + Vector256 LeftValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step]); // TODO: Is there a better way to do this? + Vector256 MiddleValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step + 1]); + Vector256 RightValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step + 2]); + + Vector256 DiffLeft = Avx.Subtract(MiddleValues, LeftValues); + Vector256 DiffRight = Avx.Subtract(MiddleValues, RightValues); + Vector256 DifferenceSum = Avx.Add(DiffLeft, DiffRight); + Vector256 Significant = Avx.Subtract(MiddleValues, Vector256.Create(0.2F)); // Avx.Subtract(Avx.Divide(DifferenceSum, MiddleValues), Vector256.Create(0.9F));// + //Vector256 ScaledDiffLeft = Avx.Divide(DiffLeft, DifferenceSum); + //Vector256 ScaledDiffRight = Avx.Divide(DiffRight, DifferenceSum); + + Vector256 PeakDetect = Avx.And(Avx.And(Avx.CompareGreaterThanOrEqual(DiffLeft, Vector256.Zero), Avx.CompareGreaterThan(DiffRight, Vector256.Zero)), Avx.CompareGreaterThanOrEqual(Significant, Vector256.Zero)); + byte PeakBitsHere = (byte)Avx.MoveMask(PeakDetect); + + PeakBitsPacked[Step / 8] = PeakBitsHere; + } + + for (int Outer = 0; Outer < PeakBitsPacked.Length; Outer++) + { + if (PeakBitsPacked[Outer] == 0) { continue; } + for (int Inner = 0; Inner < 8; Inner++) + { + if (((PeakBitsPacked[Outer] >> Inner) & 1) != 0) // Can optimize this by changing the loop var to be shifted and just anding here + { + int Index = (Outer * 8) + Inner + 1; // Offset by 1 for front padding + int SideBinsWithContent = 0; + int LeftmostSideBin = Index - 1; + int RightmostSideBin = Index + 1; + float TotalContent = RawBinValuesPadded[Index]; + float Threshold = 0.1F; // MUST be greater than 0 + while (RawBinValuesPadded[LeftmostSideBin] > Threshold) // This is guaranteed to stop at or before 0, given that RawBinValuesPadded[0] == 0 (padding intact) + { + SideBinsWithContent++; + TotalContent += RawBinValuesPadded[LeftmostSideBin]; + LeftmostSideBin--; + } + while (RawBinValuesPadded[RightmostSideBin] > Threshold) // Same but at the upper end + { + SideBinsWithContent++; + TotalContent += RawBinValuesPadded[RightmostSideBin]; + RightmostSideBin++; + } + + // Offset these over to the left, since we need the non-padded index + LeftmostSideBin--; + RightmostSideBin--; + + if (SideBinsWithContent > 4) + { + for (int i = LeftmostSideBin; i < RightmostSideBin; i++) { WidebandBitsPacked[i / 8] |= (byte)(1 << (i % 8)); } + } + } + } + } + + for (int i = 0; i < PeakBitsPacked.Length; i++) { - //Vector256 + PeakBitsPacked[i] = (byte)(PeakBitsPacked[i] & ~WidebandBitsPacked[i]); } } diff --git a/ColorChord.NET/NoteFinder/ShinNoteFinderDFT.cs b/ColorChord.NET/NoteFinder/ShinNoteFinderDFT.cs index 335f2c3..262b56d 100644 --- a/ColorChord.NET/NoteFinder/ShinNoteFinderDFT.cs +++ b/ColorChord.NET/NoteFinder/ShinNoteFinderDFT.cs @@ -131,7 +131,7 @@ public static class ShinNoteFinderDFT private static int TEMP_CycleCount = 0; - private static float IIR_CONST = 0.25F; // TODO: Scale dynamically, lower => more older data + private static float IIR_CONST = 0.85F; // TODO: Scale dynamically, lower => more older data static ShinNoteFinderDFT() { @@ -158,22 +158,24 @@ static ShinNoteFinderDFT() )] public static void Reconfigure() { - AudioBufferSizes = new uint[BinCount]; - AudioBufferSubHeads = new ushort[BinCount]; - SinTableStepSize = new DualU16[BinCount]; - SinTableLocationAdd = new DualU16[BinCount]; - SinTableLocationSub = new DualU16[BinCount]; - SinProductAccumulators = new DualI64[BinCount]; - CosProductAccumulators = new DualI64[BinCount]; - ProductAccumulators = new Vector256[BinCount]; - SmoothingFactors = new float[BinCount]; - MergedSinOutputs = new double[BinCount]; - MergedCosOutputs = new double[BinCount]; - AllBinValues = new float[BinCount + 2]; - OctaveBinValues = new float[BinsPerOctave]; - LoudnessCorrectionFactors = new float[BinCount]; - RawBinMagnitudes = new float[BinCount]; - RawBinFrequencies = new float[BinCount]; + static T[] CheckArray(T[]? array, int expectedLength) => (array == null || array.Length != expectedLength) ? new T[expectedLength] : array; + + AudioBufferSizes = CheckArray(AudioBufferSizes, BinCount); + AudioBufferSubHeads = CheckArray(AudioBufferSubHeads, BinCount); + SinTableStepSize = CheckArray(SinTableStepSize, BinCount); + SinTableLocationAdd = CheckArray(SinTableLocationAdd, BinCount); + SinTableLocationSub = CheckArray(SinTableLocationSub, BinCount); + SinProductAccumulators = CheckArray(SinProductAccumulators, BinCount); + CosProductAccumulators = CheckArray(CosProductAccumulators, BinCount); + ProductAccumulators = CheckArray(ProductAccumulators, BinCount); + SmoothingFactors = CheckArray(SmoothingFactors, BinCount); + MergedSinOutputs = CheckArray(MergedSinOutputs, BinCount); + MergedCosOutputs = CheckArray(MergedCosOutputs, BinCount); + AllBinValues = CheckArray(AllBinValues, BinCount + 2); + OctaveBinValues = CheckArray(OctaveBinValues, (int)BinsPerOctave); + LoudnessCorrectionFactors = CheckArray(LoudnessCorrectionFactors, BinCount); + RawBinMagnitudes = CheckArray(RawBinMagnitudes, BinCount); + RawBinFrequencies = CheckArray(RawBinFrequencies, BinCount); uint MaxAudioBufferSize = 0; for (uint Bin = 0; Bin < BinCount; Bin++) @@ -527,19 +529,12 @@ private static void CalculateBins(uint samplesAdded) { Vector256 PreviousSinVals = Vector256.LoadUnsafe(ref MergedSinOutputs[BinIndex]); Vector256 PreviousCosVals = Vector256.LoadUnsafe(ref MergedCosOutputs[BinIndex]); - //Vector128 IIRa32 = Vector128.LoadUnsafe(ref SmoothingFactors[BinIndex]); - //Vector256 IIRa = Avx.ConvertToVector256Double(IIRa32); - //Vector256 IIRb = Avx.ConvertToVector256Double(Sse.Subtract(Vector128.Create(1F), IIRa32)); - Vector256 NewSinVals = Vector256.LoadUnsafe(ref RotatedValues[0]); - //Vector256 MergedSinVals = Avx.Add(Avx.Multiply(PreviousSinVals, IIRb), Avx.Multiply(NewSinVals, IIRa)); - Vector256 SmoothedSinVals = Avx.Add(PreviousSinVals, NewSinVals);// Avx.Multiply(NewSinVals, IIRa)); Vector256 NewCosVals = Vector256.LoadUnsafe(ref RotatedValues[4]); - //Vector256 MergedCosVals = Avx.Add(Avx.Multiply(PreviousCosVals, IIRb), Avx.Multiply(NewCosVals, IIRa)); - Vector256 SmoothedCosVals = Avx.Add(PreviousCosVals, NewCosVals);// Avx.Multiply(NewCosVals, IIRa)); - - SmoothedSinVals.StoreUnsafe(ref MergedSinOutputs[BinIndex]); - SmoothedCosVals.StoreUnsafe(ref MergedCosOutputs[BinIndex]); + Vector256 AddedSinVals = Avx.Add(PreviousSinVals, NewSinVals); + Vector256 AddedCosVals = Avx.Add(PreviousCosVals, NewCosVals); + AddedSinVals.StoreUnsafe(ref MergedSinOutputs[BinIndex]); + AddedCosVals.StoreUnsafe(ref MergedCosOutputs[BinIndex]); } BinIndex += 4; @@ -555,6 +550,7 @@ public static void CalculateOutput() { // TODO: Having this be called asynchronously from the data input code means that the responsiveness of any IIR we apply here depends on the frequency at which this is called. // Not sure if this is a real concern. Need to think a bit more about it. + // This could be avoided by using MergedDataAmplitude to scale the IIR, but that's extra calculations if (MergedDataAmplitude == 0) { return; } int BinCount = ShinNoteFinderDFT.BinCount; Debug.Assert(BinCount % 4 == 0, "BinCount needs to be a multiple of 4."); @@ -578,13 +574,14 @@ public static void CalculateOutput() Vector256 WindowSizes = Avx.ConvertToVector256Double(Vector128.LoadUnsafe(ref AudioBufferSizes[BinIndex]).AsInt32()); Vector128 ScaledMagnitudes = Avx.ConvertToVector128Single(Avx.Divide(Avx.Sqrt(Avx.Divide(Magnitudes, WindowSizes)), Vector256.Create(3.0D))); // Scales such that a 0dB signal is approximately 1.0 // TODO: Check if this is still true - Vector128 NewMagnitudes = Sse.Add(Sse.Multiply(Vector128.LoadUnsafe(ref RawBinMagnitudes[BinIndex]), Vector128.Create(1F - IIR_CONST)), Sse.Multiply(ScaledMagnitudes, Vector128.Create(IIR_CONST))); + + Vector128 IIRa = Vector128.LoadUnsafe(ref SmoothingFactors[BinIndex]); + Vector128 IIRb = Sse.Subtract(Vector128.Create(1F), IIRa); + + Vector128 NewMagnitudes = Sse.Add(Sse.Multiply(Vector128.LoadUnsafe(ref RawBinMagnitudes[BinIndex]), IIRb), Sse.Multiply(ScaledMagnitudes, IIRa)); //ScaledMagnitudes.StoreUnsafe(ref RawBinMagnitudes[BinIndex]); NewMagnitudes.StoreUnsafe(ref RawBinMagnitudes[BinIndex]); - //Avx.Multiply(MergedSinVals, Vector256.Create((double)IIR_CONST)).StoreUnsafe(ref MergedSinOutputs[BinIndex]); - //Avx.Multiply(MergedCosVals, Vector256.Create((double)IIR_CONST)).StoreUnsafe(ref MergedCosOutputs[BinIndex]); - Vector256.Zero.StoreUnsafe(ref MergedSinOutputs[BinIndex]); Vector256.Zero.StoreUnsafe(ref MergedCosOutputs[BinIndex]); diff --git a/ColorChord.NET/Outputs/Display/Shaders/ShinNFDebug.frag b/ColorChord.NET/Outputs/Display/Shaders/ShinNFDebug.frag index 8e37c22..ddd67f7 100644 --- a/ColorChord.NET/Outputs/Display/Shaders/ShinNFDebug.frag +++ b/ColorChord.NET/Outputs/Display/Shaders/ShinNFDebug.frag @@ -1,11 +1,13 @@ #version 330 core +#extension GL_EXT_gpu_shader4 : enable #define BINS_PER_OCATVE 24 in vec2 TexCoord; uniform int BinCount; -uniform sampler2D Texture; +uniform sampler2D TextureRawBins; +uniform usampler2D TexturePeakBits, TextureWidebandBits; uniform float ScaleFactor; uniform float Exponent; @@ -18,19 +20,23 @@ vec3 HSVToRGB(vec3 c) return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y); } -vec3 AngleToRGB(float angle, float val) +vec3 AngleToRGB(float angle, float sat, float val) { float Hue; Hue = (1.0 - step(4.0 / 12.0, angle)) * ((1.0 / 3.0) - angle) * 0.5; // Yellow -> Red Hue += (step(4.0 / 12.0, angle) - step(8.0 / 12.0, angle)) * (1 - (angle - (1.0 / 3.0))); // Red -> Blue Hue += step(8.0 / 12.0, angle) * ((2.0 / 3.0) - (1.5 * (angle - (2.0 / 3.0)))); // Blue -> Yellow - return HSVToRGB(vec3(Hue, 1.0, val)); + return HSVToRGB(vec3(Hue, sat, val)); } void main() { int SectionHere = int(floor(TexCoord.x * BinCount)); - float HeightHere = pow(texture(Texture, vec2((SectionHere + 0.5) / BinCount, 0.5)).r, Exponent) * ScaleFactor; - vec3 Colour = AngleToRGB(mod(float(SectionHere) / BINS_PER_OCATVE, 1.0), 1); - FragColor = vec4(step(abs(TexCoord.y), HeightHere) * Colour, 1.0); + float HeightHere = pow(texture(TextureRawBins, vec2((SectionHere + 0.5) / BinCount, 0.5)).r, Exponent) * ScaleFactor; + uint PeakHere = (texture(TexturePeakBits, vec2(((SectionHere / 8) + 0.5) / textureSize(TexturePeakBits, 0).x, 0.5)).r >> (SectionHere % 8)) & 1u; + uint WidebandHere = (texture(TextureWidebandBits, vec2(((SectionHere / 8) + 0.5) / textureSize(TextureWidebandBits, 0).x, 0.5)).r >> (SectionHere % 8)) & 1u; + vec3 Colour = AngleToRGB(mod(float(SectionHere) / BINS_PER_OCATVE, 1.0), 1.0 - (0.6 * WidebandHere), 1.0 - (0.8 * WidebandHere)); + float IsBar = step(abs(TexCoord.y), HeightHere); + FragColor = vec4((IsBar * Colour) + ((1.0 - IsBar) * vec3(0.2) * PeakHere * Colour), 1.0); + //FragColor = vec4(vec3(0.8), 1.0); } \ No newline at end of file diff --git a/ColorChord.NET/Outputs/Display/ShinNFDebug.cs b/ColorChord.NET/Outputs/Display/ShinNFDebug.cs index 09f5b34..657a14b 100644 --- a/ColorChord.NET/Outputs/Display/ShinNFDebug.cs +++ b/ColorChord.NET/Outputs/Display/ShinNFDebug.cs @@ -27,7 +27,7 @@ public class ShinNFDebug : IDisplayMode, IConfigurableAttr -1, 1, 0, 1 // Top-Left }; - private int VertexBufferHandle, VertexArrayHandle, TextureHandle; + private int VertexBufferHandle, VertexArrayHandle, TextureHandleRawBins, TextureHandlePeakBits, TextureHandleWidebandBits; private int LocationBinCount, LocationScaleFactor, LocationExponent; private float[] RawDataIn; @@ -63,17 +63,34 @@ public void Load() this.VertexBufferHandle = GL.GenBuffer(); this.VertexArrayHandle = GL.GenVertexArray(); - this.TextureHandle = GL.GenTexture(); - GL.Uniform1(this.Shader.GetUniformLocation("Texture"), 0); + this.TextureHandleRawBins = GL.GenTexture(); + GL.Uniform1(this.Shader.GetUniformLocation("TextureRawBins"), 0); GL.ActiveTexture(TextureUnit.Texture0); - GL.BindTexture(TextureTarget.Texture2D, this.TextureHandle); - + GL.BindTexture(TextureTarget.Texture2D, this.TextureHandleRawBins); GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R32f, ColorChord.NoteFinder.AllBinValues.Length, 1, 0, PixelFormat.Red, PixelType.Float, new float[ColorChord.NoteFinder.AllBinValues.Length]); GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Nearest); GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMinFilter.Nearest); GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToBorder); GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToBorder); + this.TextureHandlePeakBits = GL.GenTexture(); + GL.Uniform1(this.Shader.GetUniformLocation("TexturePeakBits"), 1); + GL.ActiveTexture(TextureUnit.Texture1); + GL.BindTexture(TextureTarget.Texture2D, this.TextureHandlePeakBits); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Nearest); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMinFilter.Nearest); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToBorder); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToBorder); + + this.TextureHandleWidebandBits = GL.GenTexture(); + GL.Uniform1(this.Shader.GetUniformLocation("TextureWidebandBits"), 2); + GL.ActiveTexture(TextureUnit.Texture2); + GL.BindTexture(TextureTarget.Texture2D, this.TextureHandleWidebandBits); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Nearest); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMinFilter.Nearest); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToBorder); + GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToBorder); + this.LocationBinCount = this.Shader.GetUniformLocation("BinCount"); GL.Uniform1(this.LocationBinCount, ColorChord.NoteFinder.AllBinValues.Length); this.LocationScaleFactor = this.Shader.GetUniformLocation("ScaleFactor"); @@ -100,7 +117,12 @@ public void Render() if (this.RawDataIn.Length != ColorChord.NoteFinder.AllBinValues.Length) { this.RawDataIn = new float[ColorChord.NoteFinder.AllBinValues.Length]; } ColorChord.NoteFinder.AllBinValues.CopyTo(this.RawDataIn); + GL.ActiveTexture(TextureUnit.Texture0); GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R32f, ColorChord.NoteFinder!.AllBinValues.Length, 1, 0, PixelFormat.Red, PixelType.Float, this.RawDataIn); + GL.ActiveTexture(TextureUnit.Texture1); + GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R8ui, ColorChord.NoteFinder!.AllBinValues.Length / 8, 1, 0, PixelFormat.RedInteger, PixelType.UnsignedByte, ((ShinNoteFinder)ColorChord.NoteFinder).PeakBits); + GL.ActiveTexture(TextureUnit.Texture2); + GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R8ui, ColorChord.NoteFinder!.AllBinValues.Length / 8, 1, 0, PixelFormat.RedInteger, PixelType.UnsignedByte, ((ShinNoteFinder)ColorChord.NoteFinder).WidebandBits); GL.BindVertexArray(this.VertexArrayHandle); GL.DrawArrays(PrimitiveType.Triangles, 0, Geometry.Length / 2); }