Fix remote Redis eviction issues

src/CacheTower.Extensions.Redis/RedisRemoteEvictionExtension.cs

@@ -16,14 +16,20 @@ public class RedisRemoteEvictionExtension : IValueRefreshExtension
 
 		private readonly object FlaggedRefreshesLockObj = new object();
 		private HashSet<string> FlaggedRefreshes { get; }
+		private ICacheLayer[] EvictFromLayers { get; }
 
-		public RedisRemoteEvictionExtension(ConnectionMultiplexer connection, string channelPrefix = "CacheTower")
+		public RedisRemoteEvictionExtension(ConnectionMultiplexer connection, ICacheLayer[] evictFromLayers, string channelPrefix = "CacheTower")
 		{
 			if (connection == null)
 			{
 				throw new ArgumentNullException(nameof(connection));
 			}
 
+			if (evictFromLayers == null)
+			{
+				throw new ArgumentNullException(nameof(evictFromLayers));
+			}
+
 			if (channelPrefix == null)
 			{
 				throw new ArgumentNullException(nameof(channelPrefix));
@@ -32,6 +38,7 @@ public RedisRemoteEvictionExtension(ConnectionMultiplexer connection, string cha
 			Subscriber = connection.GetSubscriber();
 			RedisChannel = $"{channelPrefix}.RemoteEviction";
 			FlaggedRefreshes = new HashSet<string>(StringComparer.Ordinal);
+			EvictFromLayers = evictFromLayers;
 		}
 
 		public async ValueTask OnValueRefreshAsync(string cacheKey, TimeSpan timeToLive)
@@ -52,7 +59,7 @@ public void Register(ICacheStack cacheStack)
 			}
 			IsRegistered = true;
 
-			Subscriber.Subscribe(RedisChannel, CommandFlags.FireAndForget)
+			Subscriber.Subscribe(RedisChannel)
 				.OnMessage(async (channelMessage) =>
 				{
 					string cacheKey = channelMessage.Message;
@@ -64,7 +71,10 @@ public void Register(ICacheStack cacheStack)
 
 					if (shouldEvictLocally)
 					{
-						await cacheStack.EvictAsync(cacheKey);
+						for (var i = 0; i < EvictFromLayers.Length; i++)
+						{
+							await EvictFromLayers[i].EvictAsync(cacheKey);
+						}
 					}
 				});
 		}

tests/CacheTower.Benchmarks/Extensions/Redis/RedisRemoteEvictionExtensionBenchmark.cs

@@ -1,9 +1,7 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Attributes;
 using CacheTower.Benchmarks.Utils;
 using CacheTower.Extensions.Redis;
+using CacheTower.Providers.Memory;
 
 namespace CacheTower.Benchmarks.Extensions.Redis
 {
@@ -12,7 +10,7 @@ public class RedisRemoteEvictionExtensionBenchmark : BaseValueRefreshExtensionsB
 		[GlobalSetup]
 		public void Setup()
 		{
-			CacheExtensionProvider = () => new RedisRemoteEvictionExtension(RedisHelper.GetConnection());
+			CacheExtensionProvider = () => new RedisRemoteEvictionExtension(RedisHelper.GetConnection(), new ICacheLayer[] { new MemoryCacheLayer() });
 		}
 
 		[IterationSetup]

tests/CacheTower.Tests/Extensions/Redis/RedisRemoteEvictionExtensionTests.cs

@@ -4,6 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 using CacheTower.Extensions.Redis;
+using CacheTower.Providers.Memory;
 using CacheTower.Tests.Utils;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
@@ -16,7 +17,13 @@ public class RedisRemoteEvictionExtensionTests
 		[TestMethod, ExpectedException(typeof(ArgumentNullException))]
 		public void ThrowForNullConnection()
 		{
-			new RedisRemoteEvictionExtension(null);
+			new RedisRemoteEvictionExtension(null, new ICacheLayer[] { new MemoryCacheLayer() });
+		}
+
+		[TestMethod, ExpectedException(typeof(ArgumentNullException))]
+		public void ThrowForNullCacheEvictionLayers()
+		{
+			new RedisRemoteEvictionExtension(RedisHelper.GetConnection(), null);
 		}
 
 		[TestMethod, ExpectedException(typeof(ArgumentNullException))]
@@ -28,28 +35,33 @@ public void ThrowForNullChannel()
 		[TestMethod, ExpectedException(typeof(InvalidOperationException))]
 		public void ThrowForRegisteringMoreThanOneCacheStack()
 		{
-			var extension = new RedisRemoteEvictionExtension(RedisHelper.GetConnection());
+			var extension = new RedisRemoteEvictionExtension(RedisHelper.GetConnection(), new ICacheLayer[] { new MemoryCacheLayer() });
 			var cacheStackMock = new Mock<ICacheStack>();
 			extension.Register(cacheStackMock.Object);
 			extension.Register(cacheStackMock.Object);
 		}
 
 		[TestMethod]
-		public async Task EvictsFromChannelButNotFromRegisteredCacheStack()
+		public async Task EvictionOccursOnRefresh()
 		{
 			RedisHelper.FlushDatabase();
 
 			var connection = RedisHelper.GetConnection();
 
-			var cacheStackMock = new Mock<ICacheStack>();
-			var extension = new RedisRemoteEvictionExtension(connection);
-			extension.Register(cacheStackMock.Object);
+			var cacheStackMockOne = new Mock<ICacheStack>();
+			var cacheLayerOne = new Mock<ICacheLayer>();
+			var extensionOne = new RedisRemoteEvictionExtension(connection, new ICacheLayer[] { cacheLayerOne.Object });
+			extensionOne.Register(cacheStackMockOne.Object);
 
-			var completionSource = new TaskCompletionSource<bool>();
+			var cacheStackMockTwo = new Mock<ICacheStack>();
+			var cacheLayerTwo = new Mock<ICacheLayer>();
+			var extensionTwo = new RedisRemoteEvictionExtension(connection, new ICacheLayer[] { cacheLayerTwo.Object });
+			extensionTwo.Register(cacheStackMockTwo.Object);
 
-			await connection.GetSubscriber().SubscribeAsync("CacheTower.RemoteEviction", (channel, value) =>
+			var completionSource = new TaskCompletionSource<bool>();
+			connection.GetSubscriber().Subscribe("CacheTower.RemoteEviction").OnMessage(channelMessage =>
 			{
-				if (value == "TestKey")
+				if (channelMessage.Message == "TestKey")
 				{
 					completionSource.SetResult(true);
 				}
@@ -59,12 +71,16 @@ await connection.GetSubscriber().SubscribeAsync("CacheTower.RemoteEviction", (ch
 				}
 			});
 
-			await extension.OnValueRefreshAsync("TestKey", TimeSpan.FromDays(1));
+			await extensionOne.OnValueRefreshAsync("TestKey", TimeSpan.FromDays(1));
 
 			var succeedingTask = await Task.WhenAny(completionSource.Task, Task.Delay(TimeSpan.FromSeconds(10)));
 			Assert.AreEqual(completionSource.Task, succeedingTask, "Subscriber response took too long");
 			Assert.IsTrue(completionSource.Task.Result, "Subscribers were not notified about the refreshed value");
-			cacheStackMock.Verify(c => c.EvictAsync("TestKey"), Times.Never, "The CacheStack that published the refresh was told to evict its own cache");
+
+			await Task.Delay(500);
+
+			cacheLayerOne.Verify(c => c.EvictAsync("TestKey"), Times.Never, "Eviction took place locally where it should have been skipped");
+			cacheLayerTwo.Verify(c => c.EvictAsync("TestKey"), Times.Once, "Eviction was skipped where it should have taken place locally");
 		}
 	}
 }

tests/CacheTower.Tests/Utils/RedisHelper.cs

@@ -10,20 +10,14 @@ public static class RedisHelper
 	{
 		public static string Endpoint => Environment.GetEnvironmentVariable("REDIS_ENDPOINT") ?? "localhost:6379";
 
-		private static ConnectionMultiplexer Connection { get; set; }
-
 		public static ConnectionMultiplexer GetConnection()
 		{
-			if (Connection == null)
+			var config = new ConfigurationOptions
 			{
-				var config = new ConfigurationOptions
-				{
-					AllowAdmin = true
-				};
-				config.EndPoints.Add(Endpoint);
-				Connection = ConnectionMultiplexer.Connect(config);
-			}
-			return Connection;
+				AllowAdmin = true
+			};
+			config.EndPoints.Add(Endpoint);
+			return ConnectionMultiplexer.Connect(config);
 		}
 
 		public static void FlushDatabase()