Skip to content

Commit

Permalink
Merge pull request #52 from gfoidl/loop-boundaries
Browse files Browse the repository at this point in the history 
Loop unrolling up to next multiple of Vector<double>
  • Loading branch information
@gfoidl
gfoidl committed Apr 22, 2018
2 parents 06730b3 + 6a2f9d0 commit 5d0e3ad
Show file tree
Hide file tree
Showing 8 changed files with 251 additions and 154 deletions.
10 changes: 5 additions & 5 deletions source/gfoidl.Stochastics/Statistics/Sample.AutoCorrelation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ internal IEnumerable<double> AutoCorrelationSimd()
{
var kVec = new Vector<double>(arr, k);
var kmVec = new Vector<double>(arr, k - m);
r_xx += Vector.Dot(kVec, kmVec);
k += Vector<double>.Count;
r_xx += Vector.Dot(kVec, kmVec);
k += Vector<double>.Count;

kVec = new Vector<double>(arr, k);
kmVec = new Vector<double>(arr, k - m);
r_xx += Vector.Dot(kVec, kmVec);
k += Vector<double>.Count;
k += Vector<double>.Count;
}

for (; k < arr.Length; ++k)
Expand Down Expand Up @@ -84,7 +84,7 @@ internal double[] AutoCorrelationToArrayParallelSimd()
return corr;
}
//---------------------------------------------------------------------
private unsafe void AutoCorrelationToArrayImpl(double[] corr, (int Start, int End) range)
private unsafe void AutoCorrelationToArrayImpl(double[] corr, Range range)
{
int n = _values.Length;

Expand Down Expand Up @@ -157,7 +157,7 @@ void Core(double* a_k, double* a_km, int offset, ref double r_xx)
{
Vector<double> kVec = VectorHelper.GetVector(a_k + offset);
Vector<double> kmVec = VectorHelper.GetVector(a_km + offset);
r_xx += Vector.Dot(kVec, kmVec);
r_xx += Vector.Dot(kVec, kmVec);
}
}
#endregion
Expand Down
70 changes: 43 additions & 27 deletions source/gfoidl.Stochastics/Statistics/Sample.AverageVarianceCore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
using System.Numerics;
using System.Threading.Tasks;

#if DEBUG_ASSERT
using System.Diagnostics;
#endif

namespace gfoidl.Stochastics.Statistics
{
partial class Sample
Expand Down Expand Up @@ -55,71 +59,83 @@ private unsafe void CalculateAverageAndVarianceCoreImpl(int i, int n, out double
fixed (double* pArray = _values)
{
double* arr = pArray + i;
n -= i;
i = 0;
double* end = arr + n;

if (Vector.IsHardwareAccelerated && (n - i) >= Vector<double>.Count)
if (Vector.IsHardwareAccelerated && n >= Vector<double>.Count)
{
var avgVec = Vector<double>.Zero;

for (; i < n - 8 * Vector<double>.Count; i += 8 * Vector<double>.Count)
// https://github.com/gfoidl/Stochastics/issues/46
int m = n & ~(8 * Vector<double>.Count - 1);
for (; i < m; i += 8 * Vector<double>.Count)
{
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 1 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 2 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 3 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 4 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 5 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 6 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 7 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 1 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 2 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 3 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 4 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 5 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 6 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 7 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);

arr += 8 * Vector<double>.Count;
}

if (i < n - 4 * Vector<double>.Count)
m = n & ~(4 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 1 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 2 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 3 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 1 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 2 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 3 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);

arr += 4 * Vector<double>.Count;
i += 4 * Vector<double>.Count;
}

if (i < n - 2 * Vector<double>.Count)
m = n & ~(2 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 1 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);
Core(arr, 1 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);

arr += 2 * Vector<double>.Count;
i += 2 * Vector<double>.Count;
}

if (i < n - Vector<double>.Count)
m = n & ~(1 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance);
Core(arr, 0 * Vector<double>.Count, ref avgVec, ref tmpVariance, end);

i += Vector<double>.Count;
arr += 1 * Vector<double>.Count;
}

// Reduction -- https://github.com/gfoidl/Stochastics/issues/43
tmpAvg += avgVec.ReduceSum();
}

for (; i < n; ++i)
while (arr < end)
{
tmpAvg += pArray[i];
tmpVariance += pArray[i] * pArray[i];
tmpAvg += *arr;
tmpVariance += *arr * *arr;
arr++;
}

avg = tmpAvg;
variance = tmpVariance;
}
//-----------------------------------------------------------------
void Core(double* arr, int offset, ref Vector<double> avgVec, ref double var)
void Core(double* arr, int offset, ref Vector<double> avgVec, ref double var, double* end)
{
#if DEBUG_ASSERT
Debug.Assert(arr + offset < end);
#endif
Vector<double> vec = VectorHelper.GetVector(arr + offset);
avgVec += vec;
var += Vector.Dot(vec, vec);
avgVec += vec;
var += Vector.Dot(vec, vec);
}
}
}
Expand Down
70 changes: 44 additions & 26 deletions source/gfoidl.Stochastics/Statistics/Sample.Delta.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
using System.Numerics;
using System.Threading.Tasks;

#if DEBUG_ASSERT
using System.Diagnostics;
#endif

namespace gfoidl.Stochastics.Statistics
{
partial class Sample
Expand Down Expand Up @@ -45,67 +49,81 @@ private unsafe double CalculateDeltaImpl(int i, int n)
fixed (double* pArray = _values)
{
double* arr = pArray + i;
n -= i;
i = 0;
double* end = arr + n;

if (Vector.IsHardwareAccelerated && (n - i) >= Vector<double>.Count)
if (Vector.IsHardwareAccelerated && n >= Vector<double>.Count)
{
var avgVec = new Vector<double>(avg);
var deltaVec = new Vector<double>(0);
var deltaVec = Vector<double>.Zero;

for (; i < n - 8 * Vector<double>.Count; i += 8 * Vector<double>.Count)
// https://github.com/gfoidl/Stochastics/issues/46
int m = n & ~(8 * Vector<double>.Count - 1);
for (; i < m; i += 8 * Vector<double>.Count)
{
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 1 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 2 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 3 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 4 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 5 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 6 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 7 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 1 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 2 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 3 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 4 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 5 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 6 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 7 * Vector<double>.Count, avgVec, ref deltaVec, end);

arr += 8 * Vector<double>.Count;
}

if (i < n - 4 * Vector<double>.Count)
m = n & ~(4 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 1 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 2 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 3 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 1 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 2 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 3 * Vector<double>.Count, avgVec, ref deltaVec, end);

arr += 4 * Vector<double>.Count;
i += 4 * Vector<double>.Count;
}

if (i < n - 2 * Vector<double>.Count)
m = n & ~(2 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 1 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec, end);
Core(arr, 1 * Vector<double>.Count, avgVec, ref deltaVec, end);

arr += 2 * Vector<double>.Count;
i += 2 * Vector<double>.Count;
}

if (i < n - Vector<double>.Count)
m = n & ~(1 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec);
Core(arr, 0 * Vector<double>.Count, avgVec, ref deltaVec, end);

i += Vector<double>.Count;
arr += 1 * Vector<double>.Count;
}

// Reduction -- https://github.com/gfoidl/Stochastics/issues/43
delta += deltaVec.ReduceSum();
}

for (; i < n; ++i)
delta += Math.Abs(pArray[i] - avg);
while (arr < end)
{
delta += Math.Abs(*arr - avg);
arr++;
}
}

return delta;
//-----------------------------------------------------------------
void Core(double* arr, int offset, Vector<double> avgVec, ref Vector<double> deltaVec)
void Core(double* arr, int offset, Vector<double> avgVec, ref Vector<double> deltaVec, double* end)
{
#if DEBUG_ASSERT
Debug.Assert(arr + offset < end);
#endif
Vector<double> vec = VectorHelper.GetVector(arr + offset);
deltaVec += Vector.Abs(vec - avgVec);
deltaVec += Vector.Abs(vec - avgVec);
}
}
}
Expand Down
66 changes: 41 additions & 25 deletions source/gfoidl.Stochastics/Statistics/Sample.MinMax.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
using System.Numerics;
using System.Threading.Tasks;

#if DEBUG_ASSERT
using System.Diagnostics;
#endif

namespace gfoidl.Stochastics.Statistics
{
partial class Sample
Expand Down Expand Up @@ -54,69 +58,81 @@ private unsafe void GetMinMaxImpl(int i, int n, out double min, out double max)
fixed (double* pArray = _values)
{
double* arr = pArray + i;
n -= i;
i = 0;
double* end = arr + n;

if (Vector.IsHardwareAccelerated && (n - i) >= Vector<double>.Count)
if (Vector.IsHardwareAccelerated && n >= Vector<double>.Count)
{
var minVec = new Vector<double>(tmpMin);
var maxVec = new Vector<double>(tmpMax);

for (; i < n - 8 * Vector<double>.Count; i += 8 * Vector<double>.Count)
// https://github.com/gfoidl/Stochastics/issues/46
int m = n & ~(8 * Vector<double>.Count - 1);
for (; i < m; i += 8 * Vector<double>.Count)
{
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 1 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 2 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 3 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 4 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 5 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 6 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 7 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 1 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 2 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 3 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 4 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 5 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 6 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 7 * Vector<double>.Count, ref minVec, ref maxVec, end);

arr += 8 * Vector<double>.Count;
}

if (i < n - 4 * Vector<double>.Count)
m = n & ~(4 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 1 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 2 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 3 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 1 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 2 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 3 * Vector<double>.Count, ref minVec, ref maxVec, end);

arr += 4 * Vector<double>.Count;
i += 4 * Vector<double>.Count;
}

if (i < n - 2 * Vector<double>.Count)
m = n & ~(2 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 1 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec, end);
Core(arr, 1 * Vector<double>.Count, ref minVec, ref maxVec, end);

arr += 2 * Vector<double>.Count;
i += 2 * Vector<double>.Count;
}

if (i < n - Vector<double>.Count)
m = n & ~(1 * Vector<double>.Count - 1);
if (i < m)
{
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec);
Core(arr, 0 * Vector<double>.Count, ref minVec, ref maxVec, end);

i += Vector<double>.Count;
arr += 1 * Vector<double>.Count;
}

// Reduction
VectorHelper.ReduceMinMax(minVec, maxVec, ref tmpMin, ref tmpMax);
}

for (; i < n; ++i)
while (arr < end)
{
if (pArray[i] < tmpMin) tmpMin = pArray[i];
if (pArray[i] > tmpMax) tmpMax = pArray[i];
if (*arr < tmpMin) tmpMin = *arr;
if (*arr > tmpMax) tmpMax = *arr;
arr++;
}

min = tmpMin;
max = tmpMax;
}
//-----------------------------------------------------------------
void Core(double* arr, int offset, ref Vector<double> minVec, ref Vector<double> maxVec)
void Core(double* arr, int offset, ref Vector<double> minVec, ref Vector<double> maxVec, double* end)
{
#if DEBUG_ASSERT
Debug.Assert(arr + offset < end);
#endif
Vector<double> vec = VectorHelper.GetVector(arr + offset);
minVec = Vector.Min(minVec, vec);
maxVec = Vector.Max(maxVec, vec);
Expand Down
Loading

0 comments on commit 5d0e3ad

Please sign in to comment.