diff --git a/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs b/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs index ee2640e3..f3f7c6a3 100644 --- a/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs +++ b/BitFaster.Caching/Buffers/MpscBoundedBuffer.cs @@ -86,7 +86,7 @@ private int GetCount(int head, int tail) public BufferStatus TryAdd(T item) { int head = Volatile.Read(ref headAndTail.Head); - int tail = Volatile.Read(ref headAndTail.Tail); + int tail = headAndTail.Tail; int size = tail - head; if (size >= buffer.Length) @@ -117,7 +117,7 @@ public BufferStatus TryAdd(T item) public BufferStatus TryTake(out T item) { int head = Volatile.Read(ref headAndTail.Head); - int tail = Volatile.Read(ref headAndTail.Tail); + int tail = headAndTail.Tail; int size = tail - head; if (size == 0) @@ -136,7 +136,7 @@ public BufferStatus TryTake(out T item) return BufferStatus.Contended; } - Volatile.Write(ref buffer[index], null); + buffer[index] = null; Volatile.Write(ref this.headAndTail.Head, ++head); return BufferStatus.Success; } @@ -190,7 +190,7 @@ private int DrainToImpl(Span output) #endif { int head = Volatile.Read(ref headAndTail.Head); - int tail = Volatile.Read(ref headAndTail.Tail); + int tail = headAndTail.Tail; int size = tail - head; if (size == 0) @@ -214,13 +214,13 @@ private int DrainToImpl(Span output) break; } - Volatile.Write(ref localBuffer[index], null); + localBuffer[index] = null; Write(output, outCount++, item); head++; } while (head != tail && outCount < Length(output)); - Volatile.Write(ref this.headAndTail.Head, head); + this.headAndTail.Head = head; return outCount; } diff --git a/BitFaster.Caching/Lfu/ConcurrentLfu.cs b/BitFaster.Caching/Lfu/ConcurrentLfu.cs index c98657de..4ae642cd 100644 --- a/BitFaster.Caching/Lfu/ConcurrentLfu.cs +++ b/BitFaster.Caching/Lfu/ConcurrentLfu.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -70,7 +71,7 @@ public sealed class ConcurrentLfu : ICache, IAsyncCache, IBoun private readonly IScheduler scheduler; - private readonly LfuNode[] drainBuffer; + private readonly LfuNode[] drainBuffer; /// /// Initializes a new instance of the ConcurrentLfu class with the specified capacity. @@ -478,9 +479,10 @@ private void AfterWrite(LfuNode node) private void ScheduleAfterWrite() { var spinner = new SpinWait(); + int status = this.drainStatus.NonVolatileRead(); while (true) { - switch (this.drainStatus.Status()) + switch (status) { case DrainStatus.Idle: this.drainStatus.Cas(DrainStatus.Idle, DrainStatus.Required); @@ -494,6 +496,7 @@ private void ScheduleAfterWrite() { return; } + status = this.drainStatus.VolatileRead(); break; case DrainStatus.ProcessingToRequired: return; @@ -509,8 +512,8 @@ IEnumerator IEnumerable.GetEnumerator() private void TryScheduleDrain() { - if (this.drainStatus.Status() >= DrainStatus.ProcessingToIdle) - { + if (this.drainStatus.NonVolatileRead() >= DrainStatus.ProcessingToIdle) + { return; } @@ -521,15 +524,15 @@ private void TryScheduleDrain() if (lockTaken) { - int status = this.drainStatus.Status(); + int status = this.drainStatus.NonVolatileRead(); if (status >= DrainStatus.ProcessingToIdle) { return; } - this.drainStatus.Set(DrainStatus.ProcessingToIdle); - scheduler.Run(() => DrainBuffers()); + this.drainStatus.VolatileWrite(DrainStatus.ProcessingToIdle); + scheduler.Run(() => this.DrainBuffers()); } } finally @@ -559,7 +562,7 @@ private void DrainBuffers() } } - if (this.drainStatus.Status() == DrainStatus.Required) + if (this.drainStatus.VolatileRead() == DrainStatus.Required) { TryScheduleDrain(); } @@ -567,7 +570,7 @@ private void DrainBuffers() private bool Maintenance(LfuNode droppedWrite = null) { - this.drainStatus.Set(DrainStatus.ProcessingToIdle); + this.drainStatus.VolatileWrite(DrainStatus.ProcessingToIdle); // Note: this is only Span on .NET Core 3.1+, else this is no-op and it is still an array var buffer = this.drainBuffer.AsSpanOrArray(); @@ -609,10 +612,10 @@ private bool Maintenance(LfuNode droppedWrite = null) // 1. We drained both input buffers (all work done) // 2. or scheduler is foreground (since don't run continuously on the foreground) if ((done || !scheduler.IsBackground) && - (this.drainStatus.Status() != DrainStatus.ProcessingToIdle || + (this.drainStatus.NonVolatileRead() != DrainStatus.ProcessingToIdle || !this.drainStatus.Cas(DrainStatus.ProcessingToIdle, DrainStatus.Idle))) { - this.drainStatus.Set(DrainStatus.Required); + this.drainStatus.NonVolatileWrite(DrainStatus.Required); } return done; @@ -743,6 +746,7 @@ private ref struct EvictIterator public LfuNode node; public int freq; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public EvictIterator(CmSketch sketch, LfuNode node) { this.sketch = sketch; @@ -750,6 +754,7 @@ public EvictIterator(CmSketch sketch, LfuNode node) freq = node == null ? -1 : sketch.EstimateFrequency(node.Key); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Next() { node = node.Next; @@ -863,9 +868,10 @@ private class DrainStatus private PaddedInt drainStatus; // mutable struct, don't mark readonly + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool ShouldDrain(bool delayable) { - int status = Volatile.Read(ref this.drainStatus.Value); + int status = this.NonVolatileRead(); return status switch { Idle => !delayable, @@ -875,19 +881,34 @@ public bool ShouldDrain(bool delayable) }; } - public void Set(int newStatus) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void VolatileWrite(int newStatus) { Volatile.Write(ref this.drainStatus.Value, newStatus); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void NonVolatileWrite(int newStatus) + { + this.drainStatus.Value = newStatus; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool Cas(int oldStatus, int newStatus) { return Interlocked.CompareExchange(ref this.drainStatus.Value, newStatus, oldStatus) == oldStatus; } - public int Status() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int VolatileRead() { return Volatile.Read(ref this.drainStatus.Value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int NonVolatileRead() + { + return this.drainStatus.Value; } [ExcludeFromCodeCoverage]