Skip to content

Commit

Permalink
First attempt at peak detection and wideband signal removal, needs so…
Browse files Browse the repository at this point in the history 
…me more tweaking
  • Loading branch information
@CaiB
CaiB committed Jun 22, 2024
1 parent 7940ce7 commit e1d1426
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 45 deletions.
77 changes: 75 additions & 2 deletions ColorChord.NET/NoteFinder/ShinNoteFinder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using System.Threading;

namespace ColorChord.NET.NoteFinder;
Expand All @@ -24,6 +25,8 @@ public class ShinNoteFinder : NoteFinderCommon, ITimingSource
private int[] P_PersistentNoteIDs;
public override ReadOnlySpan<int> PersistentNoteIDs => this.P_PersistentNoteIDs;

public byte[] PeakBits, WidebandBits;

private static Thread? ProcessThread;
private static bool KeepGoing = true;

Expand All @@ -42,6 +45,8 @@ public ShinNoteFinder(string name, Dictionary<string, object> config)
P_PersistentNoteIDs = new int[NOTE_QTY];
SetupBuffers();
ShinNoteFinderDFT.Reconfigure();
PeakBits = new byte[ShinNoteFinderDFT.BinCount / 8]; // TODO: Assumes BinsPerOctave is divisible by 8, not true for 12
WidebandBits = new byte[ShinNoteFinderDFT.BinCount / 8];
}

public override int NoteCount => NOTE_QTY;
Expand Down Expand Up @@ -94,9 +99,77 @@ private static void Cycle()
public override void UpdateOutputs()
{
ShinNoteFinderDFT.CalculateOutput();
for (int i = 0; i < AllBinValues.Length; i++)

float[] RawBinValuesPadded = ShinNoteFinderDFT.AllBinValues;

Debug.Assert(RawBinValuesPadded[0] == 0F, "AllBinValues left boundary value was changed, it should always be 0.");
Debug.Assert(RawBinValuesPadded[^1] == 0F, "AllBinValues right boundary value was changed, it should always be 0.");

byte[] PeakBitsPacked = this.PeakBits;
byte[] WidebandBitsPacked = this.WidebandBits;

for (int i = 0; i < WidebandBitsPacked.Length; i++) { WidebandBitsPacked[i] = 0; }

for (int Step = 0; Step <= RawBinValuesPadded.Length - 10; Step += 8)
{
Vector256<float> LeftValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step]); // TODO: Is there a better way to do this?
Vector256<float> MiddleValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step + 1]);
Vector256<float> RightValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step + 2]);

Vector256<float> DiffLeft = Avx.Subtract(MiddleValues, LeftValues);
Vector256<float> DiffRight = Avx.Subtract(MiddleValues, RightValues);
Vector256<float> DifferenceSum = Avx.Add(DiffLeft, DiffRight);
Vector256<float> Significant = Avx.Subtract(MiddleValues, Vector256.Create(0.2F)); // Avx.Subtract(Avx.Divide(DifferenceSum, MiddleValues), Vector256.Create(0.9F));//
//Vector256<float> ScaledDiffLeft = Avx.Divide(DiffLeft, DifferenceSum);
//Vector256<float> ScaledDiffRight = Avx.Divide(DiffRight, DifferenceSum);

Vector256<float> PeakDetect = Avx.And(Avx.And(Avx.CompareGreaterThanOrEqual(DiffLeft, Vector256<float>.Zero), Avx.CompareGreaterThan(DiffRight, Vector256<float>.Zero)), Avx.CompareGreaterThanOrEqual(Significant, Vector256<float>.Zero));
byte PeakBitsHere = (byte)Avx.MoveMask(PeakDetect);

PeakBitsPacked[Step / 8] = PeakBitsHere;
}

for (int Outer = 0; Outer < PeakBitsPacked.Length; Outer++)
{
if (PeakBitsPacked[Outer] == 0) { continue; }
for (int Inner = 0; Inner < 8; Inner++)
{
if (((PeakBitsPacked[Outer] >> Inner) & 1) != 0) // Can optimize this by changing the loop var to be shifted and just anding here
{
int Index = (Outer * 8) + Inner + 1; // Offset by 1 for front padding
int SideBinsWithContent = 0;
int LeftmostSideBin = Index - 1;
int RightmostSideBin = Index + 1;
float TotalContent = RawBinValuesPadded[Index];
float Threshold = 0.1F; // MUST be greater than 0
while (RawBinValuesPadded[LeftmostSideBin] > Threshold) // This is guaranteed to stop at or before 0, given that RawBinValuesPadded[0] == 0 (padding intact)
{
SideBinsWithContent++;
TotalContent += RawBinValuesPadded[LeftmostSideBin];
LeftmostSideBin--;
}
while (RawBinValuesPadded[RightmostSideBin] > Threshold) // Same but at the upper end
{
SideBinsWithContent++;
TotalContent += RawBinValuesPadded[RightmostSideBin];
RightmostSideBin++;
}

// Offset these over to the left, since we need the non-padded index
LeftmostSideBin--;
RightmostSideBin--;

if (SideBinsWithContent > 4)
{
for (int i = LeftmostSideBin; i < RightmostSideBin; i++) { WidebandBitsPacked[i / 8] |= (byte)(1 << (i % 8)); }
}
}
}
}

for (int i = 0; i < PeakBitsPacked.Length; i++)
{
//Vector256<float>
PeakBitsPacked[i] = (byte)(PeakBitsPacked[i] & ~WidebandBitsPacked[i]);
}
}

Expand Down
61 changes: 29 additions & 32 deletions ColorChord.NET/NoteFinder/ShinNoteFinderDFT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public static class ShinNoteFinderDFT

private static int TEMP_CycleCount = 0;

private static float IIR_CONST = 0.25F; // TODO: Scale dynamically, lower => more older data
private static float IIR_CONST = 0.85F; // TODO: Scale dynamically, lower => more older data

static ShinNoteFinderDFT()
{
Expand All @@ -158,22 +158,24 @@ static ShinNoteFinderDFT()
)]
public static void Reconfigure()
{
AudioBufferSizes = new uint[BinCount];
AudioBufferSubHeads = new ushort[BinCount];
SinTableStepSize = new DualU16[BinCount];
SinTableLocationAdd = new DualU16[BinCount];
SinTableLocationSub = new DualU16[BinCount];
SinProductAccumulators = new DualI64[BinCount];
CosProductAccumulators = new DualI64[BinCount];
ProductAccumulators = new Vector256<long>[BinCount];
SmoothingFactors = new float[BinCount];
MergedSinOutputs = new double[BinCount];
MergedCosOutputs = new double[BinCount];
AllBinValues = new float[BinCount + 2];
OctaveBinValues = new float[BinsPerOctave];
LoudnessCorrectionFactors = new float[BinCount];
RawBinMagnitudes = new float[BinCount];
RawBinFrequencies = new float[BinCount];
static T[] CheckArray<T>(T[]? array, int expectedLength) => (array == null || array.Length != expectedLength) ? new T[expectedLength] : array;

AudioBufferSizes = CheckArray(AudioBufferSizes, BinCount);
AudioBufferSubHeads = CheckArray(AudioBufferSubHeads, BinCount);
SinTableStepSize = CheckArray(SinTableStepSize, BinCount);
SinTableLocationAdd = CheckArray(SinTableLocationAdd, BinCount);
SinTableLocationSub = CheckArray(SinTableLocationSub, BinCount);
SinProductAccumulators = CheckArray(SinProductAccumulators, BinCount);
CosProductAccumulators = CheckArray(CosProductAccumulators, BinCount);
ProductAccumulators = CheckArray(ProductAccumulators, BinCount);
SmoothingFactors = CheckArray(SmoothingFactors, BinCount);
MergedSinOutputs = CheckArray(MergedSinOutputs, BinCount);
MergedCosOutputs = CheckArray(MergedCosOutputs, BinCount);
AllBinValues = CheckArray(AllBinValues, BinCount + 2);
OctaveBinValues = CheckArray(OctaveBinValues, (int)BinsPerOctave);
LoudnessCorrectionFactors = CheckArray(LoudnessCorrectionFactors, BinCount);
RawBinMagnitudes = CheckArray(RawBinMagnitudes, BinCount);
RawBinFrequencies = CheckArray(RawBinFrequencies, BinCount);

uint MaxAudioBufferSize = 0;
for (uint Bin = 0; Bin < BinCount; Bin++)
Expand Down Expand Up @@ -527,19 +529,12 @@ private static void CalculateBins(uint samplesAdded)
{
Vector256<double> PreviousSinVals = Vector256.LoadUnsafe(ref MergedSinOutputs[BinIndex]);
Vector256<double> PreviousCosVals = Vector256.LoadUnsafe(ref MergedCosOutputs[BinIndex]);
//Vector128<float> IIRa32 = Vector128.LoadUnsafe(ref SmoothingFactors[BinIndex]);
//Vector256<double> IIRa = Avx.ConvertToVector256Double(IIRa32);
//Vector256<double> IIRb = Avx.ConvertToVector256Double(Sse.Subtract(Vector128.Create(1F), IIRa32));

Vector256<double> NewSinVals = Vector256.LoadUnsafe(ref RotatedValues[0]);
//Vector256<double> MergedSinVals = Avx.Add(Avx.Multiply(PreviousSinVals, IIRb), Avx.Multiply(NewSinVals, IIRa));
Vector256<double> SmoothedSinVals = Avx.Add(PreviousSinVals, NewSinVals);// Avx.Multiply(NewSinVals, IIRa));
Vector256<double> NewCosVals = Vector256.LoadUnsafe(ref RotatedValues[4]);
//Vector256<double> MergedCosVals = Avx.Add(Avx.Multiply(PreviousCosVals, IIRb), Avx.Multiply(NewCosVals, IIRa));
Vector256<double> SmoothedCosVals = Avx.Add(PreviousCosVals, NewCosVals);// Avx.Multiply(NewCosVals, IIRa));

SmoothedSinVals.StoreUnsafe(ref MergedSinOutputs[BinIndex]);
SmoothedCosVals.StoreUnsafe(ref MergedCosOutputs[BinIndex]);
Vector256<double> AddedSinVals = Avx.Add(PreviousSinVals, NewSinVals);
Vector256<double> AddedCosVals = Avx.Add(PreviousCosVals, NewCosVals);
AddedSinVals.StoreUnsafe(ref MergedSinOutputs[BinIndex]);
AddedCosVals.StoreUnsafe(ref MergedCosOutputs[BinIndex]);
}

BinIndex += 4;
Expand All @@ -555,6 +550,7 @@ public static void CalculateOutput()
{
// TODO: Having this be called asynchronously from the data input code means that the responsiveness of any IIR we apply here depends on the frequency at which this is called.
// Not sure if this is a real concern. Need to think a bit more about it.
// This could be avoided by using MergedDataAmplitude to scale the IIR, but that's extra calculations
if (MergedDataAmplitude == 0) { return; }
int BinCount = ShinNoteFinderDFT.BinCount;
Debug.Assert(BinCount % 4 == 0, "BinCount needs to be a multiple of 4.");
Expand All @@ -578,13 +574,14 @@ public static void CalculateOutput()

Vector256<double> WindowSizes = Avx.ConvertToVector256Double(Vector128.LoadUnsafe(ref AudioBufferSizes[BinIndex]).AsInt32());
Vector128<float> ScaledMagnitudes = Avx.ConvertToVector128Single(Avx.Divide(Avx.Sqrt(Avx.Divide(Magnitudes, WindowSizes)), Vector256.Create(3.0D))); // Scales such that a 0dB signal is approximately 1.0 // TODO: Check if this is still true
Vector128<float> NewMagnitudes = Sse.Add(Sse.Multiply(Vector128.LoadUnsafe(ref RawBinMagnitudes[BinIndex]), Vector128.Create(1F - IIR_CONST)), Sse.Multiply(ScaledMagnitudes, Vector128.Create(IIR_CONST)));

Vector128<float> IIRa = Vector128.LoadUnsafe(ref SmoothingFactors[BinIndex]);
Vector128<float> IIRb = Sse.Subtract(Vector128.Create(1F), IIRa);

Vector128<float> NewMagnitudes = Sse.Add(Sse.Multiply(Vector128.LoadUnsafe(ref RawBinMagnitudes[BinIndex]), IIRb), Sse.Multiply(ScaledMagnitudes, IIRa));
//ScaledMagnitudes.StoreUnsafe(ref RawBinMagnitudes[BinIndex]);
NewMagnitudes.StoreUnsafe(ref RawBinMagnitudes[BinIndex]);

//Avx.Multiply(MergedSinVals, Vector256.Create((double)IIR_CONST)).StoreUnsafe(ref MergedSinOutputs[BinIndex]);
//Avx.Multiply(MergedCosVals, Vector256.Create((double)IIR_CONST)).StoreUnsafe(ref MergedCosOutputs[BinIndex]);

Vector256<double>.Zero.StoreUnsafe(ref MergedSinOutputs[BinIndex]);
Vector256<double>.Zero.StoreUnsafe(ref MergedCosOutputs[BinIndex]);

Expand Down
18 changes: 12 additions & 6 deletions ColorChord.NET/Outputs/Display/Shaders/ShinNFDebug.frag
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
#version 330 core
#extension GL_EXT_gpu_shader4 : enable

#define BINS_PER_OCATVE 24

in vec2 TexCoord;

uniform int BinCount;
uniform sampler2D Texture;
uniform sampler2D TextureRawBins;
uniform usampler2D TexturePeakBits, TextureWidebandBits;
uniform float ScaleFactor;
uniform float Exponent;

Expand All @@ -18,19 +20,23 @@ vec3 HSVToRGB(vec3 c)
return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);
}

vec3 AngleToRGB(float angle, float val)
vec3 AngleToRGB(float angle, float sat, float val)
{
float Hue;
Hue = (1.0 - step(4.0 / 12.0, angle)) * ((1.0 / 3.0) - angle) * 0.5; // Yellow -> Red
Hue += (step(4.0 / 12.0, angle) - step(8.0 / 12.0, angle)) * (1 - (angle - (1.0 / 3.0))); // Red -> Blue
Hue += step(8.0 / 12.0, angle) * ((2.0 / 3.0) - (1.5 * (angle - (2.0 / 3.0)))); // Blue -> Yellow
return HSVToRGB(vec3(Hue, 1.0, val));
return HSVToRGB(vec3(Hue, sat, val));
}

void main()
{
int SectionHere = int(floor(TexCoord.x * BinCount));
float HeightHere = pow(texture(Texture, vec2((SectionHere + 0.5) / BinCount, 0.5)).r, Exponent) * ScaleFactor;
vec3 Colour = AngleToRGB(mod(float(SectionHere) / BINS_PER_OCATVE, 1.0), 1);
FragColor = vec4(step(abs(TexCoord.y), HeightHere) * Colour, 1.0);
float HeightHere = pow(texture(TextureRawBins, vec2((SectionHere + 0.5) / BinCount, 0.5)).r, Exponent) * ScaleFactor;
uint PeakHere = (texture(TexturePeakBits, vec2(((SectionHere / 8) + 0.5) / textureSize(TexturePeakBits, 0).x, 0.5)).r >> (SectionHere % 8)) & 1u;
uint WidebandHere = (texture(TextureWidebandBits, vec2(((SectionHere / 8) + 0.5) / textureSize(TextureWidebandBits, 0).x, 0.5)).r >> (SectionHere % 8)) & 1u;
vec3 Colour = AngleToRGB(mod(float(SectionHere) / BINS_PER_OCATVE, 1.0), 1.0 - (0.6 * WidebandHere), 1.0 - (0.8 * WidebandHere));
float IsBar = step(abs(TexCoord.y), HeightHere);
FragColor = vec4((IsBar * Colour) + ((1.0 - IsBar) * vec3(0.2) * PeakHere * Colour), 1.0);
//FragColor = vec4(vec3(0.8), 1.0);
}
32 changes: 27 additions & 5 deletions ColorChord.NET/Outputs/Display/ShinNFDebug.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class ShinNFDebug : IDisplayMode, IConfigurableAttr
-1, 1, 0, 1 // Top-Left
};

private int VertexBufferHandle, VertexArrayHandle, TextureHandle;
private int VertexBufferHandle, VertexArrayHandle, TextureHandleRawBins, TextureHandlePeakBits, TextureHandleWidebandBits;
private int LocationBinCount, LocationScaleFactor, LocationExponent;

private float[] RawDataIn;
Expand Down Expand Up @@ -63,17 +63,34 @@ public void Load()
this.VertexBufferHandle = GL.GenBuffer();
this.VertexArrayHandle = GL.GenVertexArray();

this.TextureHandle = GL.GenTexture();
GL.Uniform1(this.Shader.GetUniformLocation("Texture"), 0);
this.TextureHandleRawBins = GL.GenTexture();
GL.Uniform1(this.Shader.GetUniformLocation("TextureRawBins"), 0);
GL.ActiveTexture(TextureUnit.Texture0);
GL.BindTexture(TextureTarget.Texture2D, this.TextureHandle);

GL.BindTexture(TextureTarget.Texture2D, this.TextureHandleRawBins);
GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R32f, ColorChord.NoteFinder.AllBinValues.Length, 1, 0, PixelFormat.Red, PixelType.Float, new float[ColorChord.NoteFinder.AllBinValues.Length]);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Nearest);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMinFilter.Nearest);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToBorder);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToBorder);

this.TextureHandlePeakBits = GL.GenTexture();
GL.Uniform1(this.Shader.GetUniformLocation("TexturePeakBits"), 1);
GL.ActiveTexture(TextureUnit.Texture1);
GL.BindTexture(TextureTarget.Texture2D, this.TextureHandlePeakBits);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Nearest);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMinFilter.Nearest);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToBorder);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToBorder);

this.TextureHandleWidebandBits = GL.GenTexture();
GL.Uniform1(this.Shader.GetUniformLocation("TextureWidebandBits"), 2);
GL.ActiveTexture(TextureUnit.Texture2);
GL.BindTexture(TextureTarget.Texture2D, this.TextureHandleWidebandBits);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Nearest);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMinFilter.Nearest);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToBorder);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToBorder);

this.LocationBinCount = this.Shader.GetUniformLocation("BinCount");
GL.Uniform1(this.LocationBinCount, ColorChord.NoteFinder.AllBinValues.Length);
this.LocationScaleFactor = this.Shader.GetUniformLocation("ScaleFactor");
Expand All @@ -100,7 +117,12 @@ public void Render()

if (this.RawDataIn.Length != ColorChord.NoteFinder.AllBinValues.Length) { this.RawDataIn = new float[ColorChord.NoteFinder.AllBinValues.Length]; }
ColorChord.NoteFinder.AllBinValues.CopyTo(this.RawDataIn);
GL.ActiveTexture(TextureUnit.Texture0);
GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R32f, ColorChord.NoteFinder!.AllBinValues.Length, 1, 0, PixelFormat.Red, PixelType.Float, this.RawDataIn);
GL.ActiveTexture(TextureUnit.Texture1);
GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R8ui, ColorChord.NoteFinder!.AllBinValues.Length / 8, 1, 0, PixelFormat.RedInteger, PixelType.UnsignedByte, ((ShinNoteFinder)ColorChord.NoteFinder).PeakBits);
GL.ActiveTexture(TextureUnit.Texture2);
GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R8ui, ColorChord.NoteFinder!.AllBinValues.Length / 8, 1, 0, PixelFormat.RedInteger, PixelType.UnsignedByte, ((ShinNoteFinder)ColorChord.NoteFinder).WidebandBits);
GL.BindVertexArray(this.VertexArrayHandle);
GL.DrawArrays(PrimitiveType.Triangles, 0, Geometry.Length / 2);
}
Expand Down

0 comments on commit e1d1426

Please sign in to comment.