Skip to content

Fix multi get not working #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 15, 2017
Merged

Fix multi get not working #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3a1fcdc
Fix mistaken LogError
cnblogs-dudu Dec 13, 2017
23473c5
Fix multi get(#24)
cnblogs-dudu Dec 13, 2017
9f5c72d
Implement async multi get
cnblogs-dudu Dec 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Enyim.Caching/IMemcachedClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@ public interface IMemcachedClient : IDisposable
Task<T> GetValueAsync<T>(string key);
object Get(string key);
T Get<T>(string key);
IDictionary<string, object> Get(IEnumerable<string> keys);
IDictionary<string, T> Get<T>(IEnumerable<string> keys);
Task<IDictionary<string, T>> GetAsync<T>(IEnumerable<string> keys);

bool TryGet(string key, out object value);
bool TryGet(string key, out object value);
bool TryGetWithCas(string key, out CasResult<object> value);

CasResult<object> GetWithCas(string key);
Expand Down
20 changes: 20 additions & 0 deletions Enyim.Caching/Memcached/Transcoders/DefaultTranscoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,26 @@ T ITranscoder.Deserialize<T>(CacheItem item)
{
if (item.Data == null || item.Data.Count == 0) return default(T);

if (typeof(T).GetTypeCode() != TypeCode.Object || typeof(T) == typeof(Byte[]))
{
var value = Deserialize(item);
if (value != null)
{
if (typeof(T) == typeof(Guid))
{
return (T)(object)new Guid((string)value);
}
else
{
return (T)value;
}
}
else
{
return default(T);
}
}

using (var ms = new MemoryStream(item.Data.ToArray()))
{
using (var reader = new BsonDataReader(ms))
Expand Down
80 changes: 53 additions & 27 deletions Enyim.Caching/MemcachedClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,29 +182,9 @@ public async Task<IGetOperationResult<T>> GetAsync<T>(string key)

if (commandResult.Success)
{
if (typeof(T).GetTypeCode() == TypeCode.Object && typeof(T) != typeof(Byte[]))
{
result.Success = true;
result.Value = this.transcoder.Deserialize<T>(command.Result);
return result;
}
else
{
var tempResult = this.transcoder.Deserialize(command.Result);
if (tempResult != null)
{
result.Success = true;
if (typeof(T) == typeof(Guid))
{
result.Value = (T)(object)new Guid((string)tempResult);
}
else
{
result.Value = (T)tempResult;
}
return result;
}
}
result.Success = true;
result.Value = transcoder.Deserialize<T>(command.Result);
return result;
}
}
catch (Exception ex)
Expand Down Expand Up @@ -935,9 +915,14 @@ public async Task<bool> RemoveAsync(string key)
/// </summary>
/// <param name="keys">The list of identifiers for the items to retrieve.</param>
/// <returns>a Dictionary holding all items indexed by their key.</returns>
public IDictionary<string, object> Get(IEnumerable<string> keys)
public IDictionary<string, T> Get<T>(IEnumerable<string> keys)
{
return PerformMultiGet<object>(keys, (mget, kvp) => this.transcoder.Deserialize(kvp.Value));
return PerformMultiGet<T>(keys, (mget, kvp) => this.transcoder.Deserialize<T>(kvp.Value));
}

public async Task<IDictionary<string, T>> GetAsync<T>(IEnumerable<string> keys)
{
return await PerformMultiGetAsync<T>(keys, (mget, kvp) => this.transcoder.Deserialize<T>(kvp.Value));
}

public IDictionary<string, CasResult<object>> GetWithCas(IEnumerable<string> keys)
Expand Down Expand Up @@ -986,7 +971,6 @@ protected virtual IDictionary<string, T> PerformMultiGet<T>(IEnumerable<string>
if (hashed.TryGetValue(kvp.Key, out original))
{
var result = collector(mget, kvp);

// the lock will serialize the merge,
// but at least the commands were not waiting on each other
lock (retval) retval[original] = result;
Expand All @@ -996,7 +980,7 @@ protected virtual IDictionary<string, T> PerformMultiGet<T>(IEnumerable<string>
}
catch (Exception e)
{
_logger.LogError("PerformMultiGet", e);
_logger.LogError(0, e, "PerformMultiGet");
Copy link

@greatkeke greatkeke Dec 14, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not necessary to set EventId.

}
}));
}
Expand All @@ -1010,6 +994,48 @@ protected virtual IDictionary<string, T> PerformMultiGet<T>(IEnumerable<string>
return retval;
}

protected virtual async Task<IDictionary<string, T>> PerformMultiGetAsync<T>(IEnumerable<string> keys, Func<IMultiGetOperation, KeyValuePair<string, CacheItem>, T> collector)
{
// transform the keys and index them by hashed => original
// the mget results will be mapped using this index
var hashed = new Dictionary<string, string>();
foreach (var key in keys)
{
hashed[this.keyTransformer.Transform(key)] = key;
}

var byServer = GroupByServer(hashed.Keys);

var retval = new Dictionary<string, T>(hashed.Count);
var tasks = new List<Task>();

//execute each list of keys on their respective node
foreach (var slice in byServer)
{
var node = slice.Key;
var nodeKeys = slice.Value;
var mget = this.pool.OperationFactory.MultiGet(nodeKeys);
var task = Task.Run(async () =>
{
if ((await node.ExecuteAsync(mget)).Success)
{
foreach (var kvp in mget.Result)
{
if (hashed.TryGetValue(kvp.Key, out var original))
{
lock (retval) retval[original] = collector(mget, kvp);
}
}
}
});
tasks.Add(task);
}

await Task.WhenAll(tasks);

return retval;
}

protected Dictionary<IMemcachedNode, IList<string>> GroupByServer(IEnumerable<string> keys)
{
var retval = new Dictionary<IMemcachedNode, IList<string>>();
Expand Down
13 changes: 9 additions & 4 deletions Enyim.Caching/NullMemcachedClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,22 @@ public void Dispose()

public void FlushAll()
{
throw new NotImplementedException();

}

public IDictionary<string, object> Get(IEnumerable<string> keys)
public IDictionary<string, T> Get<T>(IEnumerable<string> keys)
{
throw new NotImplementedException();
return new Dictionary<string, T>();
}

public Task<IDictionary<string, T>> GetAsync<T>(IEnumerable<string> keys)
{
return Task.FromResult<IDictionary<string, T>>(new Dictionary<string, T>());
}

public object Get(string key)
{
throw new NotImplementedException();
return null;
}

public T Get<T>(string key)
Expand Down
21 changes: 17 additions & 4 deletions MemcachedTest/MemcachedClientTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
using System.Linq;

namespace MemcachedTest
{
Expand Down Expand Up @@ -252,27 +253,39 @@ public async Task MultiGetTest()
{
var keys = new List<string>();

for (int i = 0; i < 100; i++)
for (int i = 0; i < 10; i++)
{
string k = $"Hello_Multi_Get_{Guid.NewGuid()}_" + i;
keys.Add(k);

Assert.True(await client.StoreAsync(StoreMode.Set, k, i, DateTime.Now.AddSeconds(300)), "Store of " + k + " failed");
Assert.True(await client.StoreAsync(StoreMode.Set, k, i, DateTime.Now.AddSeconds(30)), "Store of " + k + " failed");
}

IDictionary<string, object> retvals = client.Get(keys);
IDictionary<string, int> retvals = await client.GetAsync<int>(keys);

Assert.NotEmpty(retvals);
Assert.Equal(keys.Count, retvals.Count);

object value = 0;
int value = 0;
for (int i = 0; i < keys.Count; i++)
{
string key = keys[i];

Assert.True(retvals.TryGetValue(key, out value), "missing key: " + key);
Assert.Equal(value, i);
}

var key1 = $"test_key1_{Guid.NewGuid()}";
var key2 = $"test_key2_{Guid.NewGuid()}";
var obj1 = new HashSet<int> { 1, 2 };
var obj2 = new HashSet<int> { 3, 4 };
await client.StoreAsync(StoreMode.Set, key1, obj1, DateTime.Now.AddSeconds(10));
await client.StoreAsync(StoreMode.Set, key2, obj2, DateTime.Now.AddSeconds(10));

var multiResult = await client.GetAsync<HashSet<int>>(new string[] { key1, key2 });
Assert.Equal(2, multiResult.Count);
Assert.Equal(obj1.First(), multiResult[key1].First());
Assert.Equal(obj2.First(), multiResult[key2].First());
}
}

Expand Down