From 3a1fcdca06bbd2d4591a6501b4fb33c244411e9c Mon Sep 17 00:00:00 2001 From: dudu Date: Wed, 13 Dec 2017 17:22:18 +0800 Subject: [PATCH 1/3] Fix mistaken LogError --- Enyim.Caching/MemcachedClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Enyim.Caching/MemcachedClient.cs b/Enyim.Caching/MemcachedClient.cs index 352dd6cf..88f503fc 100755 --- a/Enyim.Caching/MemcachedClient.cs +++ b/Enyim.Caching/MemcachedClient.cs @@ -996,7 +996,7 @@ protected virtual IDictionary PerformMultiGet(IEnumerable } catch (Exception e) { - _logger.LogError("PerformMultiGet", e); + _logger.LogError(0, e, "PerformMultiGet"); } })); } From 23473c5b0e739a058ddc9168a61f04c2a5a2c593 Mon Sep 17 00:00:00 2001 From: dudu Date: Wed, 13 Dec 2017 20:39:14 +0800 Subject: [PATCH 2/3] Fix multi get(#24) --- Enyim.Caching/IMemcachedClient.cs | 2 +- .../Transcoders/DefaultTranscoder.cs | 20 ++++++++++++ Enyim.Caching/MemcachedClient.cs | 32 ++++--------------- Enyim.Caching/NullMemcachedClient.cs | 8 ++--- MemcachedTest/MemcachedClientTest.cs | 21 +++++++++--- 5 files changed, 49 insertions(+), 34 deletions(-) diff --git a/Enyim.Caching/IMemcachedClient.cs b/Enyim.Caching/IMemcachedClient.cs index 543ed377..0431b722 100644 --- a/Enyim.Caching/IMemcachedClient.cs +++ b/Enyim.Caching/IMemcachedClient.cs @@ -15,7 +15,7 @@ public interface IMemcachedClient : IDisposable Task GetValueAsync(string key); object Get(string key); T Get(string key); - IDictionary Get(IEnumerable keys); + IDictionary Get(IEnumerable keys); bool TryGet(string key, out object value); bool TryGetWithCas(string key, out CasResult value); diff --git a/Enyim.Caching/Memcached/Transcoders/DefaultTranscoder.cs b/Enyim.Caching/Memcached/Transcoders/DefaultTranscoder.cs index 62aa51d0..83fdca95 100755 --- a/Enyim.Caching/Memcached/Transcoders/DefaultTranscoder.cs +++ b/Enyim.Caching/Memcached/Transcoders/DefaultTranscoder.cs @@ -32,6 +32,26 @@ T ITranscoder.Deserialize(CacheItem item) { if (item.Data == null || item.Data.Count == 0) return default(T); + if (typeof(T).GetTypeCode() != TypeCode.Object || typeof(T) == typeof(Byte[])) + { + var value = Deserialize(item); + if (value != null) + { + if (typeof(T) == typeof(Guid)) + { + return (T)(object)new Guid((string)value); + } + else + { + return (T)value; + } + } + else + { + return default(T); + } + } + using (var ms = new MemoryStream(item.Data.ToArray())) { using (var reader = new BsonDataReader(ms)) diff --git a/Enyim.Caching/MemcachedClient.cs b/Enyim.Caching/MemcachedClient.cs index 88f503fc..956f4526 100755 --- a/Enyim.Caching/MemcachedClient.cs +++ b/Enyim.Caching/MemcachedClient.cs @@ -182,29 +182,9 @@ public async Task> GetAsync(string key) if (commandResult.Success) { - if (typeof(T).GetTypeCode() == TypeCode.Object && typeof(T) != typeof(Byte[])) - { - result.Success = true; - result.Value = this.transcoder.Deserialize(command.Result); - return result; - } - else - { - var tempResult = this.transcoder.Deserialize(command.Result); - if (tempResult != null) - { - result.Success = true; - if (typeof(T) == typeof(Guid)) - { - result.Value = (T)(object)new Guid((string)tempResult); - } - else - { - result.Value = (T)tempResult; - } - return result; - } - } + result.Success = true; + result.Value = transcoder.Deserialize(command.Result); + return result; } } catch (Exception ex) @@ -935,9 +915,9 @@ public async Task RemoveAsync(string key) /// /// The list of identifiers for the items to retrieve. /// a Dictionary holding all items indexed by their key. - public IDictionary Get(IEnumerable keys) + public IDictionary Get(IEnumerable keys) { - return PerformMultiGet(keys, (mget, kvp) => this.transcoder.Deserialize(kvp.Value)); + return PerformMultiGet(keys, (mget, kvp) => this.transcoder.Deserialize(kvp.Value)); } public IDictionary> GetWithCas(IEnumerable keys) @@ -985,6 +965,8 @@ protected virtual IDictionary PerformMultiGet(IEnumerable string original; if (hashed.TryGetValue(kvp.Key, out original)) { + Console.WriteLine(kvp.Key); + Console.WriteLine(kvp.Value); var result = collector(mget, kvp); // the lock will serialize the merge, diff --git a/Enyim.Caching/NullMemcachedClient.cs b/Enyim.Caching/NullMemcachedClient.cs index 27d23343..18fcc0e9 100644 --- a/Enyim.Caching/NullMemcachedClient.cs +++ b/Enyim.Caching/NullMemcachedClient.cs @@ -80,17 +80,17 @@ public void Dispose() public void FlushAll() { - throw new NotImplementedException(); + } - public IDictionary Get(IEnumerable keys) + public IDictionary Get(IEnumerable keys) { - throw new NotImplementedException(); + return new Dictionary(); } public object Get(string key) { - throw new NotImplementedException(); + return null; } public T Get(string key) diff --git a/MemcachedTest/MemcachedClientTest.cs b/MemcachedTest/MemcachedClientTest.cs index 8db4b82c..269b4e2c 100755 --- a/MemcachedTest/MemcachedClientTest.cs +++ b/MemcachedTest/MemcachedClientTest.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System.Threading.Tasks; +using System.Linq; namespace MemcachedTest { @@ -252,20 +253,20 @@ public async Task MultiGetTest() { var keys = new List(); - for (int i = 0; i < 100; i++) + for (int i = 0; i < 10; i++) { string k = $"Hello_Multi_Get_{Guid.NewGuid()}_" + i; keys.Add(k); - Assert.True(await client.StoreAsync(StoreMode.Set, k, i, DateTime.Now.AddSeconds(300)), "Store of " + k + " failed"); + Assert.True(await client.StoreAsync(StoreMode.Set, k, i, DateTime.Now.AddSeconds(30)), "Store of " + k + " failed"); } - IDictionary retvals = client.Get(keys); + IDictionary retvals = client.Get(keys); Assert.NotEmpty(retvals); Assert.Equal(keys.Count, retvals.Count); - object value = 0; + int value = 0; for (int i = 0; i < keys.Count; i++) { string key = keys[i]; @@ -273,6 +274,18 @@ public async Task MultiGetTest() Assert.True(retvals.TryGetValue(key, out value), "missing key: " + key); Assert.Equal(value, i); } + + var key1 = $"test_key1_{Guid.NewGuid()}"; + var key2 = $"test_key2_{Guid.NewGuid()}"; + var obj1 = new HashSet { 1, 2 }; + var obj2 = new HashSet { 3, 4 }; + await client.StoreAsync(StoreMode.Set, key1, obj1, DateTime.Now.AddSeconds(10)); + await client.StoreAsync(StoreMode.Set, key2, obj2, DateTime.Now.AddSeconds(10)); + + var multiResult = client.Get>(new string[] { key1, key2 }); + Assert.Equal(2, multiResult.Count); + Assert.Equal(obj1.First(), multiResult[key1].First()); + Assert.Equal(obj2.First(), multiResult[key2].First()); } } From 9f5c72dd84b2d1102986eaee93195524e539e53c Mon Sep 17 00:00:00 2001 From: dudu Date: Wed, 13 Dec 2017 22:16:19 +0800 Subject: [PATCH 3/3] Implement async multi get --- Enyim.Caching/IMemcachedClient.cs | 3 +- Enyim.Caching/MemcachedClient.cs | 50 ++++++++++++++++++++++++++-- Enyim.Caching/NullMemcachedClient.cs | 5 +++ MemcachedTest/MemcachedClientTest.cs | 4 +-- 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/Enyim.Caching/IMemcachedClient.cs b/Enyim.Caching/IMemcachedClient.cs index 0431b722..136780c0 100644 --- a/Enyim.Caching/IMemcachedClient.cs +++ b/Enyim.Caching/IMemcachedClient.cs @@ -16,8 +16,9 @@ public interface IMemcachedClient : IDisposable object Get(string key); T Get(string key); IDictionary Get(IEnumerable keys); + Task> GetAsync(IEnumerable keys); - bool TryGet(string key, out object value); + bool TryGet(string key, out object value); bool TryGetWithCas(string key, out CasResult value); CasResult GetWithCas(string key); diff --git a/Enyim.Caching/MemcachedClient.cs b/Enyim.Caching/MemcachedClient.cs index 956f4526..000b2403 100755 --- a/Enyim.Caching/MemcachedClient.cs +++ b/Enyim.Caching/MemcachedClient.cs @@ -920,6 +920,11 @@ public IDictionary Get(IEnumerable keys) return PerformMultiGet(keys, (mget, kvp) => this.transcoder.Deserialize(kvp.Value)); } + public async Task> GetAsync(IEnumerable keys) + { + return await PerformMultiGetAsync(keys, (mget, kvp) => this.transcoder.Deserialize(kvp.Value)); + } + public IDictionary> GetWithCas(IEnumerable keys) { return PerformMultiGet>(keys, (mget, kvp) => new CasResult @@ -965,10 +970,7 @@ protected virtual IDictionary PerformMultiGet(IEnumerable string original; if (hashed.TryGetValue(kvp.Key, out original)) { - Console.WriteLine(kvp.Key); - Console.WriteLine(kvp.Value); var result = collector(mget, kvp); - // the lock will serialize the merge, // but at least the commands were not waiting on each other lock (retval) retval[original] = result; @@ -992,6 +994,48 @@ protected virtual IDictionary PerformMultiGet(IEnumerable return retval; } + protected virtual async Task> PerformMultiGetAsync(IEnumerable keys, Func, T> collector) + { + // transform the keys and index them by hashed => original + // the mget results will be mapped using this index + var hashed = new Dictionary(); + foreach (var key in keys) + { + hashed[this.keyTransformer.Transform(key)] = key; + } + + var byServer = GroupByServer(hashed.Keys); + + var retval = new Dictionary(hashed.Count); + var tasks = new List(); + + //execute each list of keys on their respective node + foreach (var slice in byServer) + { + var node = slice.Key; + var nodeKeys = slice.Value; + var mget = this.pool.OperationFactory.MultiGet(nodeKeys); + var task = Task.Run(async () => + { + if ((await node.ExecuteAsync(mget)).Success) + { + foreach (var kvp in mget.Result) + { + if (hashed.TryGetValue(kvp.Key, out var original)) + { + lock (retval) retval[original] = collector(mget, kvp); + } + } + } + }); + tasks.Add(task); + } + + await Task.WhenAll(tasks); + + return retval; + } + protected Dictionary> GroupByServer(IEnumerable keys) { var retval = new Dictionary>(); diff --git a/Enyim.Caching/NullMemcachedClient.cs b/Enyim.Caching/NullMemcachedClient.cs index 18fcc0e9..b252836c 100644 --- a/Enyim.Caching/NullMemcachedClient.cs +++ b/Enyim.Caching/NullMemcachedClient.cs @@ -88,6 +88,11 @@ public IDictionary Get(IEnumerable keys) return new Dictionary(); } + public Task> GetAsync(IEnumerable keys) + { + return Task.FromResult>(new Dictionary()); + } + public object Get(string key) { return null; diff --git a/MemcachedTest/MemcachedClientTest.cs b/MemcachedTest/MemcachedClientTest.cs index 269b4e2c..9b5c5bd8 100755 --- a/MemcachedTest/MemcachedClientTest.cs +++ b/MemcachedTest/MemcachedClientTest.cs @@ -261,7 +261,7 @@ public async Task MultiGetTest() Assert.True(await client.StoreAsync(StoreMode.Set, k, i, DateTime.Now.AddSeconds(30)), "Store of " + k + " failed"); } - IDictionary retvals = client.Get(keys); + IDictionary retvals = await client.GetAsync(keys); Assert.NotEmpty(retvals); Assert.Equal(keys.Count, retvals.Count); @@ -282,7 +282,7 @@ public async Task MultiGetTest() await client.StoreAsync(StoreMode.Set, key1, obj1, DateTime.Now.AddSeconds(10)); await client.StoreAsync(StoreMode.Set, key2, obj2, DateTime.Now.AddSeconds(10)); - var multiResult = client.Get>(new string[] { key1, key2 }); + var multiResult = await client.GetAsync>(new string[] { key1, key2 }); Assert.Equal(2, multiResult.Count); Assert.Equal(obj1.First(), multiResult[key1].First()); Assert.Equal(obj2.First(), multiResult[key2].First());