From fcdc1de8a96c75f08bd9fdec1baf57cb79cac1bb Mon Sep 17 00:00:00 2001 From: Alex Peck Date: Sun, 22 Oct 2023 18:21:59 -0700 Subject: [PATCH 1/6] test --- .../Buffers/MpscBoundedBuffer.cs | 10 ++--- BitFaster.Caching/Lfu/ConcurrentLfu.cs | 45 +++++++++++++------ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs b/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs index ee2640e3..980a514d 100644 --- a/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs +++ b/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs @@ -86,7 +86,7 @@ private int GetCount(int head, int tail) public BufferStatus TryAdd(T item) { int head = Volatile.Read(ref headAndTail.Head); - int tail = Volatile.Read(ref headAndTail.Tail); + int tail = headAndTail.Tail; int size = tail - head; if (size >= buffer.Length) @@ -117,7 +117,7 @@ public BufferStatus TryAdd(T item) public BufferStatus TryTake(out T item) { int head = Volatile.Read(ref headAndTail.Head); - int tail = Volatile.Read(ref headAndTail.Tail); + int tail = headAndTail.Tail; int size = tail - head; if (size == 0) @@ -136,7 +136,7 @@ public BufferStatus TryTake(out T item) return BufferStatus.Contended; } - Volatile.Write(ref buffer[index], null); + buffer[index] = null; Volatile.Write(ref this.headAndTail.Head, ++head); return BufferStatus.Success; } @@ -190,7 +190,7 @@ private int DrainToImpl(Span output) #endif { int head = Volatile.Read(ref headAndTail.Head); - int tail = Volatile.Read(ref headAndTail.Tail); + int tail = headAndTail.Tail; int size = tail - head; if (size == 0) @@ -214,7 +214,7 @@ private int DrainToImpl(Span output) break; } - Volatile.Write(ref localBuffer[index], null); + localBuffer[index] = null; Write(output, outCount++, item); head++; } diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs index 3c780958..d0b943bc 100644 --- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs +++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs @@ -70,7 +70,9 @@ public sealed class ConcurrentLfu : ICache, IAsyncCache, IBoun private readonly IScheduler scheduler; - private readonly LfuNode[] drainBuffer; + private readonly LfuNode[] drainBuffer; + + private readonly Action drainBuffersAction; /// /// Initializes a new instance of the ConcurrentLfu class with the specified capacity. @@ -110,7 +112,12 @@ public ConcurrentLfu(int concurrencyLevel, int capacity, IScheduler scheduler, I this.scheduler = scheduler; - this.drainBuffer = new LfuNode[this.readBuffer.Capacity]; + this.drainBuffer = new LfuNode[this.readBuffer.Capacity]; + + drainBuffersAction = () => + { + this.DrainBuffers(); + }; } // No lock count: https://arbel.net/2013/02/03/best-practices-for-using-concurrentdictionary/ @@ -480,7 +487,7 @@ private void ScheduleAfterWrite() var spinner = new SpinWait(); while (true) { - switch (this.drainStatus.Status()) + switch (this.drainStatus.VolatileRead()) { case DrainStatus.Idle: this.drainStatus.Cas(DrainStatus.Idle, DrainStatus.Required); @@ -509,7 +516,7 @@ IEnumerator IEnumerable.GetEnumerator() private void TryScheduleDrain() { - if (this.drainStatus.Status() >= DrainStatus.ProcessingToIdle) + if (this.drainStatus.VolatileRead() >= DrainStatus.ProcessingToIdle) { return; } @@ -521,15 +528,15 @@ private void TryScheduleDrain() if (lockTaken) { - int status = this.drainStatus.Status(); + int status = this.drainStatus.NonVolatileRead(); if (status >= DrainStatus.ProcessingToIdle) { return; } - this.drainStatus.Set(DrainStatus.ProcessingToIdle); - scheduler.Run(() => DrainBuffers()); + this.drainStatus.NonVolatileWrite(DrainStatus.ProcessingToIdle); + scheduler.Run(drainBuffersAction); } } finally @@ -559,7 +566,7 @@ private void DrainBuffers() } } - if (this.drainStatus.Status() == DrainStatus.Required) + if (this.drainStatus.VolatileRead() == DrainStatus.Required) { TryScheduleDrain(); } @@ -567,7 +574,7 @@ private void DrainBuffers() private bool Maintenance(LfuNode droppedWrite = null) { - this.drainStatus.Set(DrainStatus.ProcessingToIdle); + this.drainStatus.VolatileWrite(DrainStatus.ProcessingToIdle); // Note: this is only Span on .NET Core 3.1+, else this is no-op and it is still an array var buffer = this.drainBuffer.AsSpanOrArray(); @@ -609,10 +616,10 @@ private bool Maintenance(LfuNode droppedWrite = null) // 1. We drained both input buffers (all work done) // 2. or scheduler is foreground (since don't run continuously on the foreground) if ((done || !scheduler.IsBackground) && - (this.drainStatus.Status() != DrainStatus.ProcessingToIdle || + (this.drainStatus.VolatileRead() != DrainStatus.ProcessingToIdle || !this.drainStatus.Cas(DrainStatus.ProcessingToIdle, DrainStatus.Idle))) { - this.drainStatus.Set(DrainStatus.Required); + this.drainStatus.VolatileWrite(DrainStatus.Required); } return done; @@ -859,7 +866,7 @@ private class DrainStatus public bool ShouldDrain(bool delayable) { - int status = Volatile.Read(ref this.drainStatus.Value); + int status = this.drainStatus.Value; return status switch { Idle => !delayable, @@ -869,9 +876,14 @@ public bool ShouldDrain(bool delayable) }; } - public void Set(int newStatus) + public void VolatileWrite(int newStatus) { Volatile.Write(ref this.drainStatus.Value, newStatus); + } + + public void NonVolatileWrite(int newStatus) + { + this.drainStatus.Value = newStatus; } public bool Cas(int oldStatus, int newStatus) @@ -879,9 +891,14 @@ public bool Cas(int oldStatus, int newStatus) return Interlocked.CompareExchange(ref this.drainStatus.Value, newStatus, oldStatus) == oldStatus; } - public int Status() + public int VolatileRead() { return Volatile.Read(ref this.drainStatus.Value); + } + + public int NonVolatileRead() + { + return this.drainStatus.Value; } [ExcludeFromCodeCoverage] From 9914d9d160a953c877313c070b0589333bcb6983 Mon Sep 17 00:00:00 2001 From: Alex Peck Date: Sun, 22 Oct 2023 18:56:58 -0700 Subject: [PATCH 2/6] simplify+inline --- BitFaster.Caching/Lfu/ConcurrentLfu.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs index d0b943bc..a7633a64 100644 --- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs +++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -516,11 +517,6 @@ IEnumerator IEnumerable.GetEnumerator() private void TryScheduleDrain() { - if (this.drainStatus.VolatileRead() >= DrainStatus.ProcessingToIdle) - { - return; - } - bool lockTaken = false; try { @@ -864,9 +860,10 @@ private class DrainStatus private PaddedInt drainStatus; // mutable struct, don't mark readonly + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool ShouldDrain(bool delayable) { - int status = this.drainStatus.Value; + int status = this.VolatileRead(); return status switch { Idle => !delayable, @@ -876,26 +873,31 @@ public bool ShouldDrain(bool delayable) }; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void VolatileWrite(int newStatus) { Volatile.Write(ref this.drainStatus.Value, newStatus); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void NonVolatileWrite(int newStatus) { this.drainStatus.Value = newStatus; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool Cas(int oldStatus, int newStatus) { return Interlocked.CompareExchange(ref this.drainStatus.Value, newStatus, oldStatus) == oldStatus; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int VolatileRead() { return Volatile.Read(ref this.drainStatus.Value); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int NonVolatileRead() { return this.drainStatus.Value; From 491adec7d8a2fa2149c7ea872bffd5e3e3b7107c Mon Sep 17 00:00:00 2001 From: Alex Peck Date: Sun, 22 Oct 2023 23:23:45 -0700 Subject: [PATCH 3/6] rem del --- BitFaster.Caching/Lfu/ConcurrentLfu.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs index a7633a64..54b14acd 100644 --- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs +++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs @@ -72,8 +72,6 @@ public sealed class ConcurrentLfu : ICache, IAsyncCache, IBoun private readonly IScheduler scheduler; private readonly LfuNode[] drainBuffer; - - private readonly Action drainBuffersAction; /// /// Initializes a new instance of the ConcurrentLfu class with the specified capacity. @@ -113,12 +111,7 @@ public ConcurrentLfu(int concurrencyLevel, int capacity, IScheduler scheduler, I this.scheduler = scheduler; - this.drainBuffer = new LfuNode[this.readBuffer.Capacity]; - - drainBuffersAction = () => - { - this.DrainBuffers(); - }; + this.drainBuffer = new LfuNode[this.readBuffer.Capacity]; } // No lock count: https://arbel.net/2013/02/03/best-practices-for-using-concurrentdictionary/ @@ -532,7 +525,7 @@ private void TryScheduleDrain() } this.drainStatus.NonVolatileWrite(DrainStatus.ProcessingToIdle); - scheduler.Run(drainBuffersAction); + scheduler.Run(() => this.DrainBuffers()); } } finally From 507e28a2033a9be5e21fdd2f54cd50cec6583b17 Mon Sep 17 00:00:00 2001 From: Alex Peck Date: Tue, 24 Oct 2023 00:22:34 -0700 Subject: [PATCH 4/6] inline iter --- BitFaster.Caching/Lfu/ConcurrentLfu.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs index d7fa5a2e..3dde7027 100644 --- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs +++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs @@ -739,6 +739,7 @@ private ref struct EvictIterator public LfuNode node; public int freq; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public EvictIterator(CmSketch sketch, LfuNode node) { this.sketch = sketch; @@ -746,6 +747,7 @@ public EvictIterator(CmSketch sketch, LfuNode node) freq = node == null ? -1 : sketch.EstimateFrequency(node.Key); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Next() { node = node.Next; From 29a42cd190db47d6e0988f0e35fc84f1ee3bf6ec Mon Sep 17 00:00:00 2001 From: Alex Peck Date: Tue, 7 Nov 2023 18:05:54 -0800 Subject: [PATCH 5/6] match caffeine --- BitFaster.Caching/Buffers/MpscBoundedBuffer.cs | 2 +- BitFaster.Caching/Lfu/ConcurrentLfu.cs | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs b/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs index 980a514d..f3f7c6a3 100644 --- a/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs +++ b/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs @@ -220,7 +220,7 @@ private int DrainToImpl(Span output) } while (head != tail && outCount < Length(output)); - Volatile.Write(ref this.headAndTail.Head, head); + this.headAndTail.Head = head; return outCount; } diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs index 3dde7027..2b47204f 100644 --- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs +++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs @@ -481,7 +481,7 @@ private void ScheduleAfterWrite() var spinner = new SpinWait(); while (true) { - switch (this.drainStatus.VolatileRead()) + switch (this.drainStatus.NonVolatileRead()) { case DrainStatus.Idle: this.drainStatus.Cas(DrainStatus.Idle, DrainStatus.Required); @@ -510,6 +510,11 @@ IEnumerator IEnumerable.GetEnumerator() private void TryScheduleDrain() { + if (this.drainStatus.NonVolatileRead() >= DrainStatus.ProcessingToIdle) + { + return; + } + bool lockTaken = false; try { @@ -524,7 +529,7 @@ private void TryScheduleDrain() return; } - this.drainStatus.NonVolatileWrite(DrainStatus.ProcessingToIdle); + this.drainStatus.VolatileWrite(DrainStatus.ProcessingToIdle); scheduler.Run(() => this.DrainBuffers()); } } @@ -605,10 +610,10 @@ private bool Maintenance(LfuNode droppedWrite = null) // 1. We drained both input buffers (all work done) // 2. or scheduler is foreground (since don't run continuously on the foreground) if ((done || !scheduler.IsBackground) && - (this.drainStatus.VolatileRead() != DrainStatus.ProcessingToIdle || + (this.drainStatus.NonVolatileRead() != DrainStatus.ProcessingToIdle || !this.drainStatus.Cas(DrainStatus.ProcessingToIdle, DrainStatus.Idle))) { - this.drainStatus.VolatileWrite(DrainStatus.Required); + this.drainStatus.NonVolatileWrite(DrainStatus.Required); } return done; @@ -864,7 +869,7 @@ private class DrainStatus [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool ShouldDrain(bool delayable) { - int status = this.VolatileRead(); + int status = this.NonVolatileRead(); return status switch { Idle => !delayable, From f638359ba249c9276e717f4f653eed57916fa7aa Mon Sep 17 00:00:00 2001 From: Alex Peck Date: Tue, 7 Nov 2023 18:22:37 -0800 Subject: [PATCH 6/6] afterwrite --- BitFaster.Caching/Lfu/ConcurrentLfu.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs index 2b47204f..4ae642cd 100644 --- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs +++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs @@ -479,9 +479,10 @@ private void AfterWrite(LfuNode node) private void ScheduleAfterWrite() { var spinner = new SpinWait(); + int status = this.drainStatus.NonVolatileRead(); while (true) { - switch (this.drainStatus.NonVolatileRead()) + switch (status) { case DrainStatus.Idle: this.drainStatus.Cas(DrainStatus.Idle, DrainStatus.Required); @@ -495,6 +496,7 @@ private void ScheduleAfterWrite() { return; } + status = this.drainStatus.VolatileRead(); break; case DrainStatus.ProcessingToRequired: return;