diff --git a/BitFaster.Caching.HitRateAnalysis/ReadOnlySpanExtensions.cs b/BitFaster.Caching.HitRateAnalysis/ReadOnlySpanExtensions.cs new file mode 100644 index 00000000..03a372f4 --- /dev/null +++ b/BitFaster.Caching.HitRateAnalysis/ReadOnlySpanExtensions.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BitFaster.Caching.HitRateAnalysis +{ + // This was added in .NET 5 then reverted since it only splits on space + // https://github.com/dotnet/runtime/pull/295/files + public static class ReadOnlySpanExtensions + { + public static SpanSplitEnumerator Split(this ReadOnlySpan span) => new SpanSplitEnumerator(span, ' '); + + public static SpanSplitEnumerator Split(this ReadOnlySpan span, char separator) => new SpanSplitEnumerator(span, separator); + + public static SpanSplitEnumerator Split(this ReadOnlySpan span, string separator) => new SpanSplitEnumerator(span, separator ?? string.Empty); + } + + public ref struct SpanSplitEnumerator where T : IEquatable + { + private readonly ReadOnlySpan _buffer; + + private readonly ReadOnlySpan _separators; + private readonly T _separator; + + private readonly int _separatorLength; + private readonly bool _splitOnSingleToken; + + private readonly bool _isInitialized; + + private int _startCurrent; + private int _endCurrent; + private int _startNext; + + /// + /// Returns an enumerator that allows for iteration over the split span. + /// + /// Returns a that can be used to iterate over the split span. + public SpanSplitEnumerator GetEnumerator() => this; + + /// + /// Returns the current element of the enumeration. + /// + /// Returns a instance that indicates the bounds of the current element withing the source span. + public Range Current => new Range(_startCurrent, _endCurrent); + + internal SpanSplitEnumerator(ReadOnlySpan span, ReadOnlySpan separators) + { + _isInitialized = true; + _buffer = span; + _separators = separators; + _separator = default!; + _splitOnSingleToken = false; + _separatorLength = _separators.Length != 0 ? _separators.Length : 1; + _startCurrent = 0; + _endCurrent = 0; + _startNext = 0; + } + + internal SpanSplitEnumerator(ReadOnlySpan span, T separator) + { + _isInitialized = true; + _buffer = span; + _separator = separator; + _separators = default; + _splitOnSingleToken = true; + _separatorLength = 1; + _startCurrent = 0; + _endCurrent = 0; + _startNext = 0; + } + + /// + /// Advances the enumerator to the next element of the enumeration. + /// + /// if the enumerator was successfully advanced to the next element; if the enumerator has passed the end of the enumeration. + public bool MoveNext() + { + if (!_isInitialized || _startNext > _buffer.Length) + { + return false; + } + + ReadOnlySpan slice = _buffer.Slice(_startNext); + _startCurrent = _startNext; + + int separatorIndex = _splitOnSingleToken ? slice.IndexOf(_separator) : slice.IndexOf(_separators); + int elementLength = (separatorIndex != -1 ? separatorIndex : slice.Length); + + _endCurrent = _startCurrent + elementLength; + _startNext = _endCurrent + _separatorLength; + return true; + } + } +} diff --git a/BitFaster.Caching.UnitTests/Lru/ConcurrentLruTests.cs b/BitFaster.Caching.UnitTests/Lru/ConcurrentLruTests.cs index e2981b59..acdaa0e3 100644 --- a/BitFaster.Caching.UnitTests/Lru/ConcurrentLruTests.cs +++ b/BitFaster.Caching.UnitTests/Lru/ConcurrentLruTests.cs @@ -150,6 +150,14 @@ public void WhenItemsAddedEnumerateContainsKvps() enumerable.Should().BeEquivalentTo(new[] { new KeyValuePair(1, "1"), new KeyValuePair(2, "2") }); } + [Fact] + public void FromColdWarmupFillsWarmQueue() + { + this.Warmup(); + + this.lru.Count.Should().Be(9); + } + [Fact] public void WhenItemExistsTryGetReturnsValueAndTrue() { @@ -226,14 +234,12 @@ public async Task WhenDifferentKeysAreRequesteValueIsCreatedForEachAsync() [Fact] public void WhenValuesAreNotReadAndMoreKeysRequestedThanCapacityCountDoesNotIncrease() { - int hotColdCapacity = hotCap + coldCap; - for (int i = 0; i < hotColdCapacity + 1; i++) - { - lru.GetOrAdd(i, valueFactory.Create); - } + this.Warmup(); + + var result = lru.GetOrAdd(1, valueFactory.Create); - lru.Count.Should().Be(hotColdCapacity); - valueFactory.timesCalled.Should().Be(hotColdCapacity + 1); + lru.Count.Should().Be(9); + valueFactory.timesCalled.Should().Be(10); } [Fact] @@ -279,118 +285,154 @@ public void WhenKeysAreContinuouslyRequestedInTheOrderTheyAreAddedCountIsBounded [Fact] public void WhenValueIsNotTouchedAndExpiresFromHotValueIsBumpedToCold() { - lru.GetOrAdd(0, valueFactory.Create); + this.Warmup(); + + lru.GetOrAdd(0, valueFactory.Create); // Don't touch in hot + lru.GetOrAdd(1, valueFactory.Create); lru.GetOrAdd(2, valueFactory.Create); lru.GetOrAdd(3, valueFactory.Create); + lru.GetOrAdd(4, valueFactory.Create); + lru.GetOrAdd(5, valueFactory.Create); + lru.GetOrAdd(6, valueFactory.Create); + lru.GetOrAdd(7, valueFactory.Create); + lru.GetOrAdd(8, valueFactory.Create); + lru.GetOrAdd(9, valueFactory.Create); - lru.HotCount.Should().Be(3); - lru.WarmCount.Should().Be(0); - lru.ColdCount.Should().Be(1); + lru.TryGet(0, out var value).Should().BeFalse(); } [Fact] public void WhenValueIsTouchedAndExpiresFromHotValueIsBumpedToWarm() { + this.Warmup(); + lru.GetOrAdd(0, valueFactory.Create); - lru.GetOrAdd(0, valueFactory.Create); + lru.GetOrAdd(0, valueFactory.Create); // Touch in hot lru.GetOrAdd(1, valueFactory.Create); lru.GetOrAdd(2, valueFactory.Create); lru.GetOrAdd(3, valueFactory.Create); + lru.GetOrAdd(4, valueFactory.Create); + lru.GetOrAdd(5, valueFactory.Create); + lru.GetOrAdd(6, valueFactory.Create); + lru.GetOrAdd(7, valueFactory.Create); + lru.GetOrAdd(8, valueFactory.Create); + lru.GetOrAdd(9, valueFactory.Create); - lru.HotCount.Should().Be(3); - lru.WarmCount.Should().Be(1); - lru.ColdCount.Should().Be(0); + lru.TryGet(0, out var value).Should().BeTrue(); } [Fact] public void WhenValueIsTouchedAndExpiresFromColdItIsBumpedToWarm() { + this.Warmup(); + lru.GetOrAdd(0, valueFactory.Create); + lru.GetOrAdd(1, valueFactory.Create); lru.GetOrAdd(2, valueFactory.Create); - lru.GetOrAdd(3, valueFactory.Create); + lru.GetOrAdd(3, valueFactory.Create); // push 0 to cold (not touched in hot) - // touch 0 while it is in cold - lru.GetOrAdd(0, valueFactory.Create); + lru.GetOrAdd(0, valueFactory.Create); // Touch 0 in cold - lru.GetOrAdd(4, valueFactory.Create); + lru.GetOrAdd(4, valueFactory.Create); // fully cycle cold, this will evict 0 if it is not moved to warm lru.GetOrAdd(5, valueFactory.Create); lru.GetOrAdd(6, valueFactory.Create); + lru.GetOrAdd(7, valueFactory.Create); + lru.GetOrAdd(8, valueFactory.Create); + lru.GetOrAdd(9, valueFactory.Create); - lru.HotCount.Should().Be(3); - lru.WarmCount.Should().Be(1); - lru.ColdCount.Should().Be(3); + lru.TryGet(0, out var value).Should().BeTrue(); } [Fact] public void WhenValueIsNotTouchedAndExpiresFromColdItIsRemoved() { + this.Warmup(); + lru.GetOrAdd(0, valueFactory.Create); + lru.GetOrAdd(1, valueFactory.Create); lru.GetOrAdd(2, valueFactory.Create); - lru.GetOrAdd(3, valueFactory.Create); - lru.GetOrAdd(4, valueFactory.Create); + lru.GetOrAdd(3, valueFactory.Create); // push 0 to cold (not touched in hot) + + // Don't touch 0 in cold + + lru.GetOrAdd(4, valueFactory.Create); // fully cycle cold, this will evict 0 if it is not moved to warm lru.GetOrAdd(5, valueFactory.Create); lru.GetOrAdd(6, valueFactory.Create); + lru.GetOrAdd(7, valueFactory.Create); + lru.GetOrAdd(8, valueFactory.Create); + lru.GetOrAdd(9, valueFactory.Create); - // insert 7, 0th item will expire from cold - lru.HotCount.Should().Be(3); - lru.WarmCount.Should().Be(0); - lru.ColdCount.Should().Be(3); - - lru.TryGet(0, out var value).Should().Be(false); + lru.TryGet(0, out var value).Should().BeFalse(); } [Fact] public void WhenValueIsNotTouchedAndExpiresFromWarmValueIsBumpedToCold() { - // first 4 values are touched in hot, promote to warm - lru.GetOrAdd(0, valueFactory.Create); + this.Warmup(); + lru.GetOrAdd(0, valueFactory.Create); - lru.GetOrAdd(1, valueFactory.Create); + lru.GetOrAdd(0, valueFactory.Create); // Touch 0 in hot, it will promote to warm + lru.GetOrAdd(1, valueFactory.Create); lru.GetOrAdd(2, valueFactory.Create); - lru.GetOrAdd(2, valueFactory.Create); - lru.GetOrAdd(3, valueFactory.Create); - lru.GetOrAdd(3, valueFactory.Create); + lru.GetOrAdd(3, valueFactory.Create); // push 0 to warm - // 3 values added to hot fill warm - lru.GetOrAdd(4, valueFactory.Create); - lru.GetOrAdd(5, valueFactory.Create); - lru.GetOrAdd(6, valueFactory.Create); + // touch next 3 values, so they will promote to warm + lru.GetOrAdd(4, valueFactory.Create); lru.GetOrAdd(4, valueFactory.Create); + lru.GetOrAdd(5, valueFactory.Create); lru.GetOrAdd(5, valueFactory.Create); + lru.GetOrAdd(6, valueFactory.Create); lru.GetOrAdd(6, valueFactory.Create); - lru.HotCount.Should().Be(3); - lru.WarmCount.Should().Be(3); - lru.ColdCount.Should().Be(1); + // push 4,5,6 to warm, 0 to cold + lru.GetOrAdd(7, valueFactory.Create); + lru.GetOrAdd(8, valueFactory.Create); + lru.GetOrAdd(9, valueFactory.Create); + + // verify 0 is present, but don't touch it + lru.Keys.Should().Contain(0); + + // push 7,8,9 to cold, evict 0 + lru.GetOrAdd(10, valueFactory.Create); + lru.GetOrAdd(11, valueFactory.Create); + lru.GetOrAdd(12, valueFactory.Create); + + lru.TryGet(0, out var value).Should().BeFalse(); } [Fact] public void WhenValueIsTouchedAndExpiresFromWarmValueIsBumpedBackIntoWarm() { - // first 4 values are touched in hot, promote to warm - lru.GetOrAdd(0, valueFactory.Create); + this.Warmup(); + lru.GetOrAdd(0, valueFactory.Create); - lru.GetOrAdd(1, valueFactory.Create); + lru.GetOrAdd(0, valueFactory.Create); // Touch 0 in hot, it will promote to warm + lru.GetOrAdd(1, valueFactory.Create); lru.GetOrAdd(2, valueFactory.Create); - lru.GetOrAdd(2, valueFactory.Create); - lru.GetOrAdd(3, valueFactory.Create); - lru.GetOrAdd(3, valueFactory.Create); + lru.GetOrAdd(3, valueFactory.Create); // push 0 to warm - // touch 0 while it is warm - lru.GetOrAdd(0, valueFactory.Create); + // touch next 3 values, so they will promote to warm + lru.GetOrAdd(4, valueFactory.Create); lru.GetOrAdd(4, valueFactory.Create); + lru.GetOrAdd(5, valueFactory.Create); lru.GetOrAdd(5, valueFactory.Create); + lru.GetOrAdd(6, valueFactory.Create); lru.GetOrAdd(6, valueFactory.Create); - // 3 values added to hot fill warm. Only 0 is touched. - lru.GetOrAdd(4, valueFactory.Create); - lru.GetOrAdd(5, valueFactory.Create); - lru.GetOrAdd(6, valueFactory.Create); + // push 4,5,6 to warm, 0 to cold + lru.GetOrAdd(7, valueFactory.Create); + lru.GetOrAdd(8, valueFactory.Create); + lru.GetOrAdd(9, valueFactory.Create); - // When warm fills, 2 items are processed. 1 is promoted back into warm, and 1 into cold. - lru.HotCount.Should().Be(3); - lru.WarmCount.Should().Be(3); - lru.ColdCount.Should().Be(1); + // Touch 0 + lru.TryGet(0, out var value).Should().BeTrue(); + + // push 7,8,9 to cold, cycle 0 back to warm + lru.GetOrAdd(10, valueFactory.Create); + lru.GetOrAdd(11, valueFactory.Create); + lru.GetOrAdd(12, valueFactory.Create); + + lru.TryGet(0, out value).Should().BeTrue(); } [Fact] @@ -399,13 +441,20 @@ public void WhenValueExpiresItIsDisposed() var lruOfDisposable = new ConcurrentLru(1, new EqualCapacityPartition(6), EqualityComparer.Default); var disposableValueFactory = new DisposableValueFactory(); - for (int i = 0; i < 5; i++) + for (int i = 0; i < 7; i++) { lruOfDisposable.GetOrAdd(i, disposableValueFactory.Create); } - disposableValueFactory.Items[0].IsDisposed.Should().BeTrue(); + disposableValueFactory.Items[0].IsDisposed.Should().BeFalse(); disposableValueFactory.Items[1].IsDisposed.Should().BeFalse(); + + disposableValueFactory.Items[2].IsDisposed.Should().BeTrue(); + + disposableValueFactory.Items[3].IsDisposed.Should().BeFalse(); + disposableValueFactory.Items[4].IsDisposed.Should().BeFalse(); + disposableValueFactory.Items[5].IsDisposed.Should().BeFalse(); + disposableValueFactory.Items[6].IsDisposed.Should().BeFalse(); } [Fact] @@ -414,19 +463,23 @@ public void WhenValueEvictedItemRemovedEventIsFired() var lruEvents = new ConcurrentLru(1, new EqualCapacityPartition(6), EqualityComparer.Default); lruEvents.ItemRemoved += OnLruItemRemoved; - for (int i = 0; i < 6; i++) + // First 6 adds + // hot[6, 5], warm[2, 1], cold[4, 3] + // => + // hot[8, 7], warm[1, 0], cold[6, 5], evicted[4, 3] + for (int i = 0; i < 8; i++) { lruEvents.GetOrAdd(i + 1, i => i + 1); } removedItems.Count.Should().Be(2); - removedItems[0].Key.Should().Be(1); - removedItems[0].Value.Should().Be(2); + removedItems[0].Key.Should().Be(3); + removedItems[0].Value.Should().Be(4); removedItems[0].Reason.Should().Be(ItemRemovedReason.Evicted); - removedItems[1].Key.Should().Be(2); - removedItems[1].Value.Should().Be(3); + removedItems[1].Key.Should().Be(4); + removedItems[1].Value.Should().Be(5); removedItems[1].Reason.Should().Be(ItemRemovedReason.Evicted); } @@ -578,7 +631,7 @@ public void WhenKeyDoesNotExistAddOrUpdateMaintainsLruOrder() lru.AddOrUpdate(4, "4"); lru.HotCount.Should().Be(3); - lru.ColdCount.Should().Be(1); // items must have been enqueued and cycled for one of them to reach the cold queue + lru.WarmCount.Should().Be(1); // items must have been enqueued and cycled for one of them to reach the warm queue } [Fact] @@ -834,5 +887,18 @@ public void WhenItemsAreTrimmedAnEventIsFired() removedItems[1].Value.Should().Be(3); removedItems[1].Reason.Should().Be(ItemRemovedReason.Trimmed); } + + private void Warmup() + { + lru.GetOrAdd(-1, valueFactory.Create); + lru.GetOrAdd(-2, valueFactory.Create); + lru.GetOrAdd(-3, valueFactory.Create); + lru.GetOrAdd(-4, valueFactory.Create); + lru.GetOrAdd(-5, valueFactory.Create); + lru.GetOrAdd(-6, valueFactory.Create); + lru.GetOrAdd(-7, valueFactory.Create); + lru.GetOrAdd(-8, valueFactory.Create); + lru.GetOrAdd(-9, valueFactory.Create); + } } } diff --git a/BitFaster.Caching.UnitTests/Lru/ConcurrentTLruTests.cs b/BitFaster.Caching.UnitTests/Lru/ConcurrentTLruTests.cs index ba58e0c0..258de271 100644 --- a/BitFaster.Caching.UnitTests/Lru/ConcurrentTLruTests.cs +++ b/BitFaster.Caching.UnitTests/Lru/ConcurrentTLruTests.cs @@ -88,19 +88,23 @@ public void WhenValueEvictedItemRemovedEventIsFired() var lruEvents = new ConcurrentTLru(1, new EqualCapacityPartition(6), EqualityComparer.Default, timeToLive); lruEvents.ItemRemoved += OnLruItemRemoved; - for (int i = 0; i < 6; i++) + // First 6 adds + // hot[6, 5], warm[2, 1], cold[4, 3] + // => + // hot[8, 7], warm[1, 0], cold[6, 5], evicted[4, 3] + for (int i = 0; i < 8; i++) { lruEvents.GetOrAdd(i + 1, i => i + 1); } removedItems.Count.Should().Be(2); - removedItems[0].Key.Should().Be(1); - removedItems[0].Value.Should().Be(2); + removedItems[0].Key.Should().Be(3); + removedItems[0].Value.Should().Be(4); removedItems[0].Reason.Should().Be(ItemRemovedReason.Evicted); - removedItems[1].Key.Should().Be(2); - removedItems[1].Value.Should().Be(3); + removedItems[1].Key.Should().Be(4); + removedItems[1].Value.Should().Be(5); removedItems[1].Reason.Should().Be(ItemRemovedReason.Evicted); } @@ -165,7 +169,7 @@ public async Task WhenCacheHasExpiredAndFreshItemsExpireRemovesOnlyExpiredItems( lru.AddOrUpdate(5, "5"); lru.AddOrUpdate(6, "6"); - await Task.Delay(timeToLive * 2); + await Task.Delay(timeToLive * 4); lru.GetOrAdd(1, valueFactory.Create); lru.GetOrAdd(2, valueFactory.Create); diff --git a/BitFaster.Caching/Lru/TemplateConcurrentLru.cs b/BitFaster.Caching/Lru/TemplateConcurrentLru.cs index 5e4488b3..16024f24 100644 --- a/BitFaster.Caching/Lru/TemplateConcurrentLru.cs +++ b/BitFaster.Caching/Lru/TemplateConcurrentLru.cs @@ -47,6 +47,7 @@ public class TemplateConcurrentLru : ICache, IEnumerable this.capacity.Hot) + { + Interlocked.Decrement(ref this.hotCount); + + if (this.hotQueue.TryDequeue(out var item)) + { + // always move to warm until it is full + if (this.warmCount < this.capacity.Warm) + { + // If there is a race, we will potentially add multiple items to warm. Guard by cycling the queue. + this.Move(item, ItemDestination.Warm, ItemRemovedReason.Evicted); + CycleWarm(); + } + else + { + // Else mark isWarm and move items to cold. + // If there is a race, we will potentially add multiple items to cold. Guard by cycling the queue. + Volatile.Write(ref this.isWarm, true); + this.Move(item, ItemDestination.Cold, ItemRemovedReason.Evicted); + CycleCold(); + } + } + else + { + Interlocked.Increment(ref this.hotCount); + } + } } private void CycleHot()