From 24f7852949cc5ccabbd0f42dbfad17a56d7ddd92 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Mon, 17 Jun 2019 21:39:11 +0800 Subject: [PATCH 01/12] :sparkles: Prepare Disk caching --- EasyCaching.sln | 7 + build/releasenotes.props | 3 + build/version.props | 1 + .../Configurations/DiskOptions.cs | 11 ++ .../Configurations/DiskOptionsExtension.cs | 12 ++ .../EasyCachingOptionsExtensions.cs | 23 +++ .../DefaultDiskCachingProvider.cs | 164 ++++++++++++++++++ src/EasyCaching.Disk/EasyCaching.Disk.csproj | 39 +++++ src/EasyCaching.Disk/Internal/DiskCaching.cs | 9 + src/EasyCaching.Disk/Internal/IDiskCaching.cs | 7 + 10 files changed, 276 insertions(+) create mode 100644 src/EasyCaching.Disk/Configurations/DiskOptions.cs create mode 100644 src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs create mode 100644 src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs create mode 100644 src/EasyCaching.Disk/DefaultDiskCachingProvider.cs create mode 100644 src/EasyCaching.Disk/EasyCaching.Disk.csproj create mode 100644 src/EasyCaching.Disk/Internal/DiskCaching.cs create mode 100644 src/EasyCaching.Disk/Internal/IDiskCaching.cs diff --git a/EasyCaching.sln b/EasyCaching.sln index e8f9f0fc..8dc5aa75 100644 --- a/EasyCaching.sln +++ b/EasyCaching.sln @@ -49,6 +49,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EasyCaching.CSRedis", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EasyCaching.Bus.CSRedis", "src\EasyCaching.Bus.CSRedis\EasyCaching.Bus.CSRedis.csproj", "{861E5373-BEF6-4AA2-92C7-8F4941A079E7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EasyCaching.Disk", "src\EasyCaching.Disk\EasyCaching.Disk.csproj", "{3D48FD75-01D6-44F9-B7C3-CB6DE784F476}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -139,6 +141,10 @@ Global {861E5373-BEF6-4AA2-92C7-8F4941A079E7}.Debug|Any CPU.Build.0 = Debug|Any CPU {861E5373-BEF6-4AA2-92C7-8F4941A079E7}.Release|Any CPU.ActiveCfg = Release|Any CPU {861E5373-BEF6-4AA2-92C7-8F4941A079E7}.Release|Any CPU.Build.0 = Release|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {CE61FAA2-0233-451C-991D-4222ED61C84B} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} @@ -162,5 +168,6 @@ Global {6EBE36A2-F128-4C63-B90A-B700D8C2F2E8} = {EBB55F65-7D07-4281-8D5E-7B0CA88E1AD0} {6584761E-E51C-408F-BE51-CA0F6269589B} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} {861E5373-BEF6-4AA2-92C7-8F4941A079E7} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} EndGlobalSection EndGlobal diff --git a/build/releasenotes.props b/build/releasenotes.props index 2654fab8..774164da 100644 --- a/build/releasenotes.props +++ b/build/releasenotes.props @@ -45,5 +45,8 @@ 1. Upgrading dependencies. + + 1. init. + diff --git a/build/version.props b/build/version.props index 2fc070ba..6a373beb 100644 --- a/build/version.props +++ b/build/version.props @@ -16,5 +16,6 @@ 0.5.6 0.5.6 0.5.6 + 0.5.6 diff --git a/src/EasyCaching.Disk/Configurations/DiskOptions.cs b/src/EasyCaching.Disk/Configurations/DiskOptions.cs new file mode 100644 index 00000000..9b26b198 --- /dev/null +++ b/src/EasyCaching.Disk/Configurations/DiskOptions.cs @@ -0,0 +1,11 @@ +namespace EasyCaching.Disk +{ + using EasyCaching.Core.Configurations; + + public class DiskOptions : BaseProviderOptions + { + public DiskOptions() + { + } + } +} diff --git a/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs b/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs new file mode 100644 index 00000000..a8129a16 --- /dev/null +++ b/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs @@ -0,0 +1,12 @@ +namespace EasyCaching.Disk +{ + using EasyCaching.Core.Configurations; + + public static class DiskOptionsExtension + { + public static EasyCachingOptions UseDisk(this EasyCachingOptions options, string name = "") + { + return null; + } + } +} diff --git a/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs b/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs new file mode 100644 index 00000000..05e53d6b --- /dev/null +++ b/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs @@ -0,0 +1,23 @@ +namespace EasyCaching.Disk +{ + using EasyCaching.Core.Configurations; + using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; + + internal sealed class EasyCachingOptionsExtensions : IEasyCachingOptionsExtension + { + public EasyCachingOptionsExtensions() + { + } + + public void AddServices(IServiceCollection services) + { + throw new System.NotImplementedException(); + } + + public void WithServices(IApplicationBuilder app) + { + throw new System.NotImplementedException(); + } + } +} diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs new file mode 100644 index 00000000..e5e43670 --- /dev/null +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -0,0 +1,164 @@ +namespace EasyCaching.Disk +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using EasyCaching.Core; + + public class DefaultDiskCachingProvider : EasyCachingAbstractProvider + { + public DefaultDiskCachingProvider() + { + } + + public override bool BaseExists(string cacheKey) + { + throw new NotImplementedException(); + } + + public override Task BaseExistsAsync(string cacheKey) + { + throw new NotImplementedException(); + } + + public override void BaseFlush() + { + throw new NotImplementedException(); + } + + public override Task BaseFlushAsync() + { + throw new NotImplementedException(); + } + + public override CacheValue BaseGet(string cacheKey, Func dataRetriever, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override CacheValue BaseGet(string cacheKey) + { + throw new NotImplementedException(); + } + + public override IDictionary> BaseGetAll(IEnumerable cacheKeys) + { + throw new NotImplementedException(); + } + + public override Task>> BaseGetAllAsync(IEnumerable cacheKeys) + { + throw new NotImplementedException(); + } + + public override Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override Task BaseGetAsync(string cacheKey, Type type) + { + throw new NotImplementedException(); + } + + public override Task> BaseGetAsync(string cacheKey) + { + throw new NotImplementedException(); + } + + public override IDictionary> BaseGetByPrefix(string prefix) + { + throw new NotImplementedException(); + } + + public override Task>> BaseGetByPrefixAsync(string prefix) + { + throw new NotImplementedException(); + } + + public override int BaseGetCount(string prefix = "") + { + throw new NotImplementedException(); + } + + public override TimeSpan BaseGetExpiration(string cacheKey) + { + throw new NotImplementedException(); + } + + public override Task BaseGetExpirationAsync(string cacheKey) + { + throw new NotImplementedException(); + } + + public override void BaseRefresh(string cacheKey, T cacheValue, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override void BaseRemove(string cacheKey) + { + throw new NotImplementedException(); + } + + public override void BaseRemoveAll(IEnumerable cacheKeys) + { + throw new NotImplementedException(); + } + + public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) + { + throw new NotImplementedException(); + } + + public override Task BaseRemoveAsync(string cacheKey) + { + throw new NotImplementedException(); + } + + public override void BaseRemoveByPrefix(string prefix) + { + throw new NotImplementedException(); + } + + public override Task BaseRemoveByPrefixAsync(string prefix) + { + throw new NotImplementedException(); + } + + public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override void BaseSetAll(IDictionary values, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override Task BaseSetAllAsync(IDictionary values, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expiration) + { + throw new NotImplementedException(); + } + + public override Task BaseTrySetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/EasyCaching.Disk/EasyCaching.Disk.csproj b/src/EasyCaching.Disk/EasyCaching.Disk.csproj new file mode 100644 index 00000000..9010896e --- /dev/null +++ b/src/EasyCaching.Disk/EasyCaching.Disk.csproj @@ -0,0 +1,39 @@ + + + + + netstandard2.0 + Catcher Wong + Catcher Wong + $(EasyCachingDiskPackageVersion) + + A simple disk(file) caching provider. + + In-Memory,LocalCache,Caching,Cache + https://github.com/dotnetcore/EasyCaching + https://github.com/dotnetcore/EasyCaching/blob/master/LICENSE + https://github.com/dotnetcore/EasyCaching + https://github.com/dotnetcore/EasyCaching + https://raw.githubusercontent.com/dotnetcore/EasyCaching/master/media/nuget-icon.png + + $(EasyCachingDiskPackageNotes) + + + + + true + $(NoWarn);1591 + + + + + + + + + + + + + + diff --git a/src/EasyCaching.Disk/Internal/DiskCaching.cs b/src/EasyCaching.Disk/Internal/DiskCaching.cs new file mode 100644 index 00000000..7d539f48 --- /dev/null +++ b/src/EasyCaching.Disk/Internal/DiskCaching.cs @@ -0,0 +1,9 @@ +namespace EasyCaching.Disk +{ + public class DiskCaching : IDiskCaching + { + public DiskCaching() + { + } + } +} diff --git a/src/EasyCaching.Disk/Internal/IDiskCaching.cs b/src/EasyCaching.Disk/Internal/IDiskCaching.cs new file mode 100644 index 00000000..3f76adc2 --- /dev/null +++ b/src/EasyCaching.Disk/Internal/IDiskCaching.cs @@ -0,0 +1,7 @@ +namespace EasyCaching.Disk +{ + public interface IDiskCaching + { + + } +} From fa6729d024072bc66734c024f6825d5e24dadaeb Mon Sep 17 00:00:00 2001 From: catcherwong Date: Wed, 19 Jun 2019 23:02:10 +0800 Subject: [PATCH 02/12] :construction: Set/Get/Exists --- .../Internal/DiskCacheValue.cs | 21 ++++++ src/EasyCaching.Disk/Internal/DiskCaching.cs | 65 ++++++++++++++++++- .../Internal/DiskDbOptions.cs | 7 ++ src/EasyCaching.Disk/Internal/IDiskCaching.cs | 8 ++- 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 src/EasyCaching.Disk/Internal/DiskCacheValue.cs create mode 100644 src/EasyCaching.Disk/Internal/DiskDbOptions.cs diff --git a/src/EasyCaching.Disk/Internal/DiskCacheValue.cs b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs new file mode 100644 index 00000000..07da7bab --- /dev/null +++ b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs @@ -0,0 +1,21 @@ +namespace EasyCaching.Disk +{ + using System; + using MessagePack; + + public class DiskCacheValue + { + [SerializationConstructor] + public DiskCacheValue(byte[] val, int second) + { + Value = val; + Expiration = DateTimeOffset.UtcNow.AddSeconds(second); + } + + [Key(0)] + public byte[] Value { get; private set; } + + [Key(1)] + public DateTimeOffset Expiration { get; private set; } + } +} diff --git a/src/EasyCaching.Disk/Internal/DiskCaching.cs b/src/EasyCaching.Disk/Internal/DiskCaching.cs index 7d539f48..4bf4710a 100644 --- a/src/EasyCaching.Disk/Internal/DiskCaching.cs +++ b/src/EasyCaching.Disk/Internal/DiskCaching.cs @@ -1,9 +1,72 @@ namespace EasyCaching.Disk { + using System; + using System.IO; + using EasyCaching.Core; + using MessagePack; + public class DiskCaching : IDiskCaching { - public DiskCaching() + private readonly DiskDbOptions _options; + private readonly string _name; + + public DiskCaching(string name, DiskDbOptions optionsAccessor) { + ArgumentCheck.NotNull(optionsAccessor, nameof(optionsAccessor)); + + _name = name; + _options = optionsAccessor; + } + + public bool Exists(string key) + { + var path = GetFilePath(key); + + if (!File.Exists(path)) + { + return false; + } + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var val = MessagePackSerializer.Deserialize(stream); + + return val.Expiration > DateTimeOffset.UtcNow; + } + } + + public byte[] Get(string key) + { + var path = GetFilePath(key); + + if (!File.Exists(path)) return null; + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var bytes = MessagePackSerializer.Deserialize(stream); + return bytes.Value; + } + } + + public bool Set(string key, byte[] value, TimeSpan expiresIn) + { + var path = GetFilePath(key); + + var cached = new DiskCacheValue(value, (int)expiresIn.TotalSeconds); + + var bytes = MessagePackSerializer.Serialize(cached); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + { + stream.Write(bytes, 0, bytes.Length); + return true; + } + } + + private string GetFilePath(string key) + { + var path = Path.Combine(_options.BasePath, $"{key}.dat"); + return path; } } } diff --git a/src/EasyCaching.Disk/Internal/DiskDbOptions.cs b/src/EasyCaching.Disk/Internal/DiskDbOptions.cs new file mode 100644 index 00000000..b96dbf17 --- /dev/null +++ b/src/EasyCaching.Disk/Internal/DiskDbOptions.cs @@ -0,0 +1,7 @@ +namespace EasyCaching.Disk +{ + public class DiskDbOptions + { + public string BasePath { get; set; } + } +} diff --git a/src/EasyCaching.Disk/Internal/IDiskCaching.cs b/src/EasyCaching.Disk/Internal/IDiskCaching.cs index 3f76adc2..3ae3c4f7 100644 --- a/src/EasyCaching.Disk/Internal/IDiskCaching.cs +++ b/src/EasyCaching.Disk/Internal/IDiskCaching.cs @@ -1,7 +1,13 @@ namespace EasyCaching.Disk { + using System; + public interface IDiskCaching { - + bool Set(string key, byte[] value, TimeSpan expiresIn); + + bool Exists(string key); + + byte[] Get(string key); } } From 6a6c0bd228f546a1c1bdce322fe1ecd123f36ba5 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Thu, 20 Jun 2019 23:09:33 +0800 Subject: [PATCH 03/12] :construction: WIP --- .../Configurations/DiskOptions.cs | 2 + .../DefaultDiskCachingProvider.cs | 94 ++++++++- .../Internal/DiskCacheValue.cs | 1 + src/EasyCaching.Disk/Internal/DiskCaching.cs | 190 +++++++++++------- src/EasyCaching.Disk/Internal/IDiskCaching.cs | 25 ++- 5 files changed, 226 insertions(+), 86 deletions(-) diff --git a/src/EasyCaching.Disk/Configurations/DiskOptions.cs b/src/EasyCaching.Disk/Configurations/DiskOptions.cs index 9b26b198..d37cc3c9 100644 --- a/src/EasyCaching.Disk/Configurations/DiskOptions.cs +++ b/src/EasyCaching.Disk/Configurations/DiskOptions.cs @@ -7,5 +7,7 @@ public class DiskOptions : BaseProviderOptions public DiskOptions() { } + + public DiskDbOptions DBConfig { get; set; } = new DiskDbOptions(); } } diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index e5e43670..4a1ffae5 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -2,18 +2,53 @@ { using System; using System.Collections.Generic; + using System.IO; using System.Threading.Tasks; using EasyCaching.Core; + using MessagePack; + using Microsoft.Extensions.Logging; public class DefaultDiskCachingProvider : EasyCachingAbstractProvider { + /// + /// The options. + /// + private readonly DiskOptions _options; + + /// + /// The logger. + /// + private readonly ILogger _logger; + + /// + /// The cache stats. + /// + private readonly CacheStats _cacheStats; + + /// + /// The name. + /// + private readonly string _name; + public DefaultDiskCachingProvider() { } public override bool BaseExists(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) + { + return false; + } + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var val = MessagePackSerializer.Deserialize(stream); + + return val.Expiration > DateTimeOffset.UtcNow; + } } public override Task BaseExistsAsync(string cacheKey) @@ -38,7 +73,24 @@ public override CacheValue BaseGet(string cacheKey, Func dataRetriever, public override CacheValue BaseGet(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) return CacheValue.Null; + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = MessagePackSerializer.Deserialize(stream); + + if(cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + return new CacheValue(t, true); + } + else + { + return CacheValue.NoValue; + } + } } public override IDictionary> BaseGetAll(IEnumerable cacheKeys) @@ -103,7 +155,23 @@ public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan public override void BaseRemove(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(key); + + if (!File.Exists(path)) + { + return; + //return true; + } + + try + { + File.Delete(path); + //return true; + } + catch + { + //return false; + } } public override void BaseRemoveAll(IEnumerable cacheKeys) @@ -133,7 +201,19 @@ public override Task BaseRemoveByPrefixAsync(string prefix) public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expiration) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + var value = MessagePackSerializer.Serialize(cacheValue); + + var cached = new DiskCacheValue(value, (int)expiration.TotalSeconds); + + var bytes = MessagePackSerializer.Serialize(cached); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + { + stream.Write(bytes, 0, bytes.Length); + //return true; + } } public override void BaseSetAll(IDictionary values, TimeSpan expiration) @@ -160,5 +240,11 @@ public override Task BaseTrySetAsync(string cacheKey, T cacheValue, Tim { throw new NotImplementedException(); } + + private string GetFilePath(string key) + { + var path = Path.Combine(_options.DBConfig.BasePath, $"{key}.dat"); + return path; + } } } diff --git a/src/EasyCaching.Disk/Internal/DiskCacheValue.cs b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs index 07da7bab..4cfdeaa0 100644 --- a/src/EasyCaching.Disk/Internal/DiskCacheValue.cs +++ b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs @@ -12,6 +12,7 @@ public DiskCacheValue(byte[] val, int second) Expiration = DateTimeOffset.UtcNow.AddSeconds(second); } + [Key(0)] public byte[] Value { get; private set; } diff --git a/src/EasyCaching.Disk/Internal/DiskCaching.cs b/src/EasyCaching.Disk/Internal/DiskCaching.cs index 4bf4710a..feca0699 100644 --- a/src/EasyCaching.Disk/Internal/DiskCaching.cs +++ b/src/EasyCaching.Disk/Internal/DiskCaching.cs @@ -1,72 +1,118 @@ -namespace EasyCaching.Disk -{ - using System; - using System.IO; - using EasyCaching.Core; - using MessagePack; - - public class DiskCaching : IDiskCaching - { - private readonly DiskDbOptions _options; - private readonly string _name; - - public DiskCaching(string name, DiskDbOptions optionsAccessor) - { - ArgumentCheck.NotNull(optionsAccessor, nameof(optionsAccessor)); - - _name = name; - _options = optionsAccessor; - } - - public bool Exists(string key) - { - var path = GetFilePath(key); - - if (!File.Exists(path)) - { - return false; - } - - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var val = MessagePackSerializer.Deserialize(stream); - - return val.Expiration > DateTimeOffset.UtcNow; - } - } - - public byte[] Get(string key) - { - var path = GetFilePath(key); - - if (!File.Exists(path)) return null; - - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var bytes = MessagePackSerializer.Deserialize(stream); - return bytes.Value; - } - } - - public bool Set(string key, byte[] value, TimeSpan expiresIn) - { - var path = GetFilePath(key); - - var cached = new DiskCacheValue(value, (int)expiresIn.TotalSeconds); - - var bytes = MessagePackSerializer.Serialize(cached); - - using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) - { - stream.Write(bytes, 0, bytes.Length); - return true; - } - } - - private string GetFilePath(string key) - { - var path = Path.Combine(_options.BasePath, $"{key}.dat"); - return path; - } - } -} +//namespace EasyCaching.Disk +//{ +// using System; +// using System.Collections.Generic; +// using System.IO; +// using EasyCaching.Core; +// using MessagePack; + +// public class DiskCaching : IDiskCaching +// { +// private readonly DiskDbOptions _options; +// private readonly string _name; + +// public DiskCaching(string name, DiskDbOptions optionsAccessor) +// { +// ArgumentCheck.NotNull(optionsAccessor, nameof(optionsAccessor)); + +// _name = name; +// _options = optionsAccessor; +// } + +// public bool Exists(string key) +// { +// var path = GetFilePath(key); + +// if (!File.Exists(path)) +// { +// return false; +// } + +// using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) +// { +// var val = MessagePackSerializer.Deserialize(stream); + +// return val.Expiration > DateTimeOffset.UtcNow; +// } +// } + +// public byte[] Get(string key) +// { +// var path = GetFilePath(key); + +// if (!File.Exists(path)) return null; + +// using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) +// { +// var bytes = MessagePackSerializer.Deserialize(stream); +// return bytes.Value; +// } +// } + +// public bool Remove(string key) +// { +// var path = GetFilePath(key); + +// if (!File.Exists(path)) return true; + +// try +// { +// File.Delete(path); +// return true; +// } +// catch +// { +// return false; +// } +// } + +// public bool Set(string key, byte[] value, TimeSpan expiresIn) +// { +// var path = GetFilePath(key); + +// var cached = new DiskCacheValue(value, (int)expiresIn.TotalSeconds); + +// var bytes = MessagePackSerializer.Serialize(cached); + +// using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) +// { +// stream.Write(bytes, 0, bytes.Length); +// return true; +// } +// } + +// public int SetAll(IDictionary values, TimeSpan expiresIn) +// { +// int i = 0; +// foreach (var item in values) +// { +// try +// { +// var path = GetFilePath(item.Key); + +// var cached = new DiskCacheValue(item.Value, (int)expiresIn.TotalSeconds); + +// var bytes = MessagePackSerializer.Serialize(cached); + +// using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) +// { +// stream.Write(bytes, 0, bytes.Length); +// } + +// i++; +// } +// catch +// { + +// } +// } +// return i; +// } + +// private string GetFilePath(string key) +// { +// var path = Path.Combine(_options.BasePath, $"{key}.dat"); +// return path; +// } +// } +//} diff --git a/src/EasyCaching.Disk/Internal/IDiskCaching.cs b/src/EasyCaching.Disk/Internal/IDiskCaching.cs index 3ae3c4f7..3ffa64ac 100644 --- a/src/EasyCaching.Disk/Internal/IDiskCaching.cs +++ b/src/EasyCaching.Disk/Internal/IDiskCaching.cs @@ -1,13 +1,18 @@ -namespace EasyCaching.Disk -{ - using System; +//namespace EasyCaching.Disk +//{ +// using System; +// using System.Collections.Generic; - public interface IDiskCaching - { - bool Set(string key, byte[] value, TimeSpan expiresIn); +// public interface IDiskCaching +// { +// bool Set(string key, byte[] value, TimeSpan expiresIn); - bool Exists(string key); +// int SetAll(IDictionary values, TimeSpan expiresIn); - byte[] Get(string key); - } -} +// bool Exists(string key); + +// byte[] Get(string key); + +// bool Remove(string key); +// } +//} From af3c3d921a4b4db7c68a2d4e31332676676ff061 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Sat, 22 Jun 2019 10:55:33 +0800 Subject: [PATCH 04/12] :construction: WIP Disk provider --- .../DefaultDiskCachingProvider.cs | 356 ++++++++++++++++-- src/EasyCaching.Disk/Internal/DiskCaching.cs | 118 ------ src/EasyCaching.Disk/Internal/IDiskCaching.cs | 18 - 3 files changed, 330 insertions(+), 162 deletions(-) delete mode 100644 src/EasyCaching.Disk/Internal/DiskCaching.cs delete mode 100644 src/EasyCaching.Disk/Internal/IDiskCaching.cs diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index 4a1ffae5..bb07811b 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -30,8 +30,22 @@ public class DefaultDiskCachingProvider : EasyCachingAbstractProvider /// private readonly string _name; - public DefaultDiskCachingProvider() + public DefaultDiskCachingProvider(string name, + DiskOptions options, + ILoggerFactory loggerFactory = null) { + this._name = name; + //this._cache = cache.Single(x => x.ProviderName == _name); + this._options = options; + this._logger = loggerFactory?.CreateLogger(); + + this._cacheStats = new CacheStats(); + + this.ProviderName = _name; + this.ProviderType = CachingProviderType.Ext1; + this.ProviderStats = _cacheStats; + this.ProviderMaxRdSecond = _options.MaxRdSecond; + this.IsDistributedProvider = false; } public override bool BaseExists(string cacheKey) @@ -51,9 +65,21 @@ public override bool BaseExists(string cacheKey) } } - public override Task BaseExistsAsync(string cacheKey) + public override async Task BaseExistsAsync(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) + { + return false; + } + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var val = await MessagePackSerializer.DeserializeAsync(stream); + + return val.Expiration > DateTimeOffset.UtcNow; + } } public override void BaseFlush() @@ -68,7 +94,56 @@ public override Task BaseFlushAsync() public override CacheValue BaseGet(string cacheKey, Func dataRetriever, TimeSpan expiration) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (File.Exists(path)) + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = MessagePackSerializer.Deserialize(stream); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + return new CacheValue(t, true); + } + } + } + + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + // TODO: how to add mutex key here + //if (!_cache.Add($"{cacheKey}_Lock", 1, TimeSpan.FromMilliseconds(_options.LockMs))) + //{ + // System.Threading.Thread.Sleep(_options.SleepMs); + // return Get(cacheKey, dataRetriever, expiration); + //} + + var res = dataRetriever(); + + if (res != null) + { + Set(cacheKey, res, expiration); + //remove mutex key + //_cache.Remove($"{cacheKey}_Lock"); + + return new CacheValue(res, true); + } + else + { + //remove mutex key + //_cache.Remove($"{cacheKey}_Lock"); + return CacheValue.NoValue; + } } public override CacheValue BaseGet(string cacheKey) @@ -103,19 +178,110 @@ public override Task>> BaseGetAllAsync(IEnu throw new NotImplementedException(); } - public override Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) + public override async Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (File.Exists(path)) + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = await MessagePackSerializer.DeserializeAsync(stream); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + return new CacheValue(t, true); + } + } + } + + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + // TODO: how to add mutex key here + //if (!_cache.Add($"{cacheKey}_Lock", 1, TimeSpan.FromMilliseconds(_options.LockMs))) + //{ + // System.Threading.Thread.Sleep(_options.SleepMs); + // return Get(cacheKey, dataRetriever, expiration); + //} + + var res = await dataRetriever(); + + if (res != null) + { + Set(cacheKey, res, expiration); + //remove mutex key + //_cache.Remove($"{cacheKey}_Lock"); + + return new CacheValue(res, true); + } + else + { + //remove mutex key + //_cache.Remove($"{cacheKey}_Lock"); + return CacheValue.NoValue; + } } - public override Task BaseGetAsync(string cacheKey, Type type) + public override async Task BaseGetAsync(string cacheKey, Type type) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) + { + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + return null; + } + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = await MessagePackSerializer.DeserializeAsync(stream); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.NonGeneric.Deserialize(type, cached.Value); + return t; + } + else + { + return null; + } + } } - public override Task> BaseGetAsync(string cacheKey) + public override async Task> BaseGetAsync(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) return CacheValue.Null; + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = await MessagePackSerializer.DeserializeAsync(stream); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + return new CacheValue(t, true); + } + else + { + return CacheValue.NoValue; + } + } } public override IDictionary> BaseGetByPrefix(string prefix) @@ -135,27 +301,52 @@ public override int BaseGetCount(string prefix = "") public override TimeSpan BaseGetExpiration(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) + { + return TimeSpan.Zero; + } + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = MessagePackSerializer.Deserialize(stream); + + return cached.Expiration.Subtract(DateTimeOffset.UtcNow); + } } - public override Task BaseGetExpirationAsync(string cacheKey) + public override async Task BaseGetExpirationAsync(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) + { + return TimeSpan.Zero; + } + + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = await MessagePackSerializer.DeserializeAsync(stream); + + return cached.Expiration.Subtract(DateTimeOffset.UtcNow); + } } public override void BaseRefresh(string cacheKey, T cacheValue, TimeSpan expiration) { - throw new NotImplementedException(); + // Obsolete } public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan expiration) { - throw new NotImplementedException(); + // Obsolete + return Task.CompletedTask; } public override void BaseRemove(string cacheKey) { - var path = GetFilePath(key); + var path = GetFilePath(cacheKey); if (!File.Exists(path)) { @@ -175,18 +366,76 @@ public override void BaseRemove(string cacheKey) } public override void BaseRemoveAll(IEnumerable cacheKeys) - { - throw new NotImplementedException(); + { + foreach (string key in cacheKeys) + { + if (string.IsNullOrWhiteSpace(key)) + continue; + + var path = GetFilePath(key); + + if (!File.Exists(path)) + { + continue; + } + + try + { + File.Delete(path); + } + catch + { + } + } } public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) { - throw new NotImplementedException(); + foreach (string key in cacheKeys) + { + if (string.IsNullOrWhiteSpace(key)) + continue; + + var path = GetFilePath(key); + + if (!File.Exists(path)) + { + continue; + } + + try + { + File.Delete(path); + } + catch + { + } + } + + return Task.CompletedTask; } public override Task BaseRemoveAsync(string cacheKey) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + if (!File.Exists(path)) + { + return Task.CompletedTask; + //return true; + } + + try + { + File.Delete(path); + //return true; + } + catch + { + //return false; + } + + return Task.CompletedTask; } public override void BaseRemoveByPrefix(string prefix) @@ -218,17 +467,71 @@ public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expirati public override void BaseSetAll(IDictionary values, TimeSpan expiration) { - throw new NotImplementedException(); + foreach (var item in values) + { + try + { + var path = GetFilePath(item.Key); + + var val = MessagePackSerializer.Serialize(item.Value); + + var cached = new DiskCacheValue(val, (int)expiration.TotalSeconds); + + var bytes = MessagePackSerializer.Serialize(cached); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + { + stream.Write(bytes, 0, bytes.Length); + } + } + catch + { + + } + } } - public override Task BaseSetAllAsync(IDictionary values, TimeSpan expiration) + public override async Task BaseSetAllAsync(IDictionary values, TimeSpan expiration) { - throw new NotImplementedException(); + foreach (var item in values) + { + try + { + var path = GetFilePath(item.Key); + + var val = MessagePackSerializer.Serialize(item.Value); + + var cached = new DiskCacheValue(val, (int)expiration.TotalSeconds); + + var bytes = MessagePackSerializer.Serialize(cached); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + } + } + catch + { + + } + } } - public override Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) { - throw new NotImplementedException(); + var path = GetFilePath(cacheKey); + + var value = MessagePackSerializer.Serialize(cacheValue); + + var cached = new DiskCacheValue(value, (int)expiration.TotalSeconds); + + var bytes = MessagePackSerializer.Serialize(cached); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + //return true; + } } public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expiration) @@ -243,7 +546,8 @@ public override Task BaseTrySetAsync(string cacheKey, T cacheValue, Tim private string GetFilePath(string key) { - var path = Path.Combine(_options.DBConfig.BasePath, $"{key}.dat"); + // TODO: Special characters for file name + var path = Path.Combine(_options.DBConfig.BasePath, _name, $"{key}.dat"); return path; } } diff --git a/src/EasyCaching.Disk/Internal/DiskCaching.cs b/src/EasyCaching.Disk/Internal/DiskCaching.cs deleted file mode 100644 index feca0699..00000000 --- a/src/EasyCaching.Disk/Internal/DiskCaching.cs +++ /dev/null @@ -1,118 +0,0 @@ -//namespace EasyCaching.Disk -//{ -// using System; -// using System.Collections.Generic; -// using System.IO; -// using EasyCaching.Core; -// using MessagePack; - -// public class DiskCaching : IDiskCaching -// { -// private readonly DiskDbOptions _options; -// private readonly string _name; - -// public DiskCaching(string name, DiskDbOptions optionsAccessor) -// { -// ArgumentCheck.NotNull(optionsAccessor, nameof(optionsAccessor)); - -// _name = name; -// _options = optionsAccessor; -// } - -// public bool Exists(string key) -// { -// var path = GetFilePath(key); - -// if (!File.Exists(path)) -// { -// return false; -// } - -// using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) -// { -// var val = MessagePackSerializer.Deserialize(stream); - -// return val.Expiration > DateTimeOffset.UtcNow; -// } -// } - -// public byte[] Get(string key) -// { -// var path = GetFilePath(key); - -// if (!File.Exists(path)) return null; - -// using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) -// { -// var bytes = MessagePackSerializer.Deserialize(stream); -// return bytes.Value; -// } -// } - -// public bool Remove(string key) -// { -// var path = GetFilePath(key); - -// if (!File.Exists(path)) return true; - -// try -// { -// File.Delete(path); -// return true; -// } -// catch -// { -// return false; -// } -// } - -// public bool Set(string key, byte[] value, TimeSpan expiresIn) -// { -// var path = GetFilePath(key); - -// var cached = new DiskCacheValue(value, (int)expiresIn.TotalSeconds); - -// var bytes = MessagePackSerializer.Serialize(cached); - -// using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) -// { -// stream.Write(bytes, 0, bytes.Length); -// return true; -// } -// } - -// public int SetAll(IDictionary values, TimeSpan expiresIn) -// { -// int i = 0; -// foreach (var item in values) -// { -// try -// { -// var path = GetFilePath(item.Key); - -// var cached = new DiskCacheValue(item.Value, (int)expiresIn.TotalSeconds); - -// var bytes = MessagePackSerializer.Serialize(cached); - -// using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) -// { -// stream.Write(bytes, 0, bytes.Length); -// } - -// i++; -// } -// catch -// { - -// } -// } -// return i; -// } - -// private string GetFilePath(string key) -// { -// var path = Path.Combine(_options.BasePath, $"{key}.dat"); -// return path; -// } -// } -//} diff --git a/src/EasyCaching.Disk/Internal/IDiskCaching.cs b/src/EasyCaching.Disk/Internal/IDiskCaching.cs deleted file mode 100644 index 3ffa64ac..00000000 --- a/src/EasyCaching.Disk/Internal/IDiskCaching.cs +++ /dev/null @@ -1,18 +0,0 @@ -//namespace EasyCaching.Disk -//{ -// using System; -// using System.Collections.Generic; - -// public interface IDiskCaching -// { -// bool Set(string key, byte[] value, TimeSpan expiresIn); - -// int SetAll(IDictionary values, TimeSpan expiresIn); - -// bool Exists(string key); - -// byte[] Get(string key); - -// bool Remove(string key); -// } -//} From 93a76e7650418bd90623ff1b69b8d69b78ef5c03 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Sun, 23 Jun 2019 08:31:30 +0800 Subject: [PATCH 05/12] :construction: WIP Disk provider --- .../DefaultDiskCachingProvider.cs | 597 ++++++++++++------ 1 file changed, 415 insertions(+), 182 deletions(-) diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index bb07811b..b6ccae86 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -1,14 +1,18 @@ namespace EasyCaching.Disk { using System; + using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; + using System.Linq; + using System.Security.Cryptography; + using System.Text; using System.Threading.Tasks; using EasyCaching.Core; using MessagePack; using Microsoft.Extensions.Logging; - public class DefaultDiskCachingProvider : EasyCachingAbstractProvider + public class DefaultDiskCachingProvider : EasyCachingAbstractProvider { /// /// The options. @@ -30,15 +34,19 @@ public class DefaultDiskCachingProvider : EasyCachingAbstractProvider /// private readonly string _name; + private readonly ConcurrentDictionary _cacheKeysMap; + + public DefaultDiskCachingProvider(string name, DiskOptions options, ILoggerFactory loggerFactory = null) { this._name = name; - //this._cache = cache.Single(x => x.ProviderName == _name); this._options = options; this._logger = loggerFactory?.CreateLogger(); + this._cacheKeysMap = new ConcurrentDictionary(); + this._cacheStats = new CacheStats(); this.ProviderName = _name; @@ -46,73 +54,97 @@ public DefaultDiskCachingProvider(string name, this.ProviderStats = _cacheStats; this.ProviderMaxRdSecond = _options.MaxRdSecond; this.IsDistributedProvider = false; + + InitCacheKey(); } - public override bool BaseExists(string cacheKey) + private void InitCacheKey() { - var path = GetFilePath(cacheKey); + var path = BuildRawPath("key"); - if (!File.Exists(path)) - { - return false; - } + if (!File.Exists(path)) return; - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + for (int i = 0; i < 3; i++) { - var val = MessagePackSerializer.Deserialize(stream); + try + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + _cacheKeysMap.TryAdd(line, GetMd5Str(line)); + } + } + } + + break; + } + catch + { - return val.Expiration > DateTimeOffset.UtcNow; + } } } + public override bool BaseExists(string cacheKey) + { + var path = BuildMd5Path(cacheKey); + + if (!File.Exists(path)) return false; + + var val = GetDiskCacheValue(path); + + return val.Expiration > DateTimeOffset.UtcNow; + } + public override async Task BaseExistsAsync(string cacheKey) { - var path = GetFilePath(cacheKey); + var path = BuildMd5Path(cacheKey); - if (!File.Exists(path)) - { - return false; - } + if (!File.Exists(path)) return false; - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var val = await MessagePackSerializer.DeserializeAsync(stream); + var val = await GetDiskCacheValueAsync(path); - return val.Expiration > DateTimeOffset.UtcNow; - } + return val.Expiration > DateTimeOffset.UtcNow; } public override void BaseFlush() { - throw new NotImplementedException(); + var path = Path.Combine(_options.DBConfig.BasePath, _name); + + Directory.Delete(path, true); } public override Task BaseFlushAsync() { - throw new NotImplementedException(); + var path = Path.Combine(_options.DBConfig.BasePath, _name); + + Directory.Delete(path, true); + + return Task.CompletedTask; } public override CacheValue BaseGet(string cacheKey, Func dataRetriever, TimeSpan expiration) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (File.Exists(path)) { - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var cached = MessagePackSerializer.Deserialize(stream); + var cached = GetDiskCacheValue(path); - if (cached.Expiration > DateTimeOffset.UtcNow) - { - var t = MessagePackSerializer.Deserialize(cached.Value); + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); - CacheStats.OnHit(); + CacheStats.OnHit(); - return new CacheValue(t, true); - } + return new CacheValue(t, true); } } @@ -122,11 +154,11 @@ public override CacheValue BaseGet(string cacheKey, Func dataRetriever, _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); // TODO: how to add mutex key here - //if (!_cache.Add($"{cacheKey}_Lock", 1, TimeSpan.FromMilliseconds(_options.LockMs))) - //{ - // System.Threading.Thread.Sleep(_options.SleepMs); - // return Get(cacheKey, dataRetriever, expiration); - //} + if (!_cacheKeysMap.TryAdd($"{cacheKey}_Lock", "1")) + { + System.Threading.Thread.Sleep(_options.SleepMs); + return Get(cacheKey, dataRetriever, expiration); + } var res = dataRetriever(); @@ -134,37 +166,34 @@ public override CacheValue BaseGet(string cacheKey, Func dataRetriever, { Set(cacheKey, res, expiration); //remove mutex key - //_cache.Remove($"{cacheKey}_Lock"); + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); return new CacheValue(res, true); } else { //remove mutex key - //_cache.Remove($"{cacheKey}_Lock"); + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); return CacheValue.NoValue; } } public override CacheValue BaseGet(string cacheKey) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) return CacheValue.Null; - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var cached = MessagePackSerializer.Deserialize(stream); + var cached = GetDiskCacheValue(path); - if(cached.Expiration > DateTimeOffset.UtcNow) - { - var t = MessagePackSerializer.Deserialize(cached.Value); - return new CacheValue(t, true); - } - else - { - return CacheValue.NoValue; - } + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + return new CacheValue(t, true); + } + else + { + return CacheValue.NoValue; } } @@ -180,25 +209,22 @@ public override Task>> BaseGetAllAsync(IEnu public override async Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (File.Exists(path)) { - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var cached = await MessagePackSerializer.DeserializeAsync(stream); + var cached = await GetDiskCacheValueAsync(path); - if (cached.Expiration > DateTimeOffset.UtcNow) - { - var t = MessagePackSerializer.Deserialize(cached.Value); + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); - CacheStats.OnHit(); + CacheStats.OnHit(); - return new CacheValue(t, true); - } + return new CacheValue(t, true); } } @@ -208,33 +234,32 @@ public override async Task> BaseGetAsync(string cacheKey, Func< _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); // TODO: how to add mutex key here - //if (!_cache.Add($"{cacheKey}_Lock", 1, TimeSpan.FromMilliseconds(_options.LockMs))) - //{ - // System.Threading.Thread.Sleep(_options.SleepMs); - // return Get(cacheKey, dataRetriever, expiration); - //} + if (!_cacheKeysMap.TryAdd($"{cacheKey}_Lock", "1")) + { + System.Threading.Thread.Sleep(_options.SleepMs); + return await GetAsync(cacheKey, dataRetriever, expiration); + } var res = await dataRetriever(); if (res != null) { - Set(cacheKey, res, expiration); + await SetAsync(cacheKey, res, expiration); //remove mutex key - //_cache.Remove($"{cacheKey}_Lock"); - + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); return new CacheValue(res, true); } else { //remove mutex key - //_cache.Remove($"{cacheKey}_Lock"); + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); return CacheValue.NoValue; } } public override async Task BaseGetAsync(string cacheKey, Type type) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) { @@ -246,52 +271,126 @@ public override async Task BaseGetAsync(string cacheKey, Type type) return null; } - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) { - var cached = await MessagePackSerializer.DeserializeAsync(stream); + var t = MessagePackSerializer.NonGeneric.Deserialize(type, cached.Value); + return t; + } + else + { + return null; + } + } - if (cached.Expiration > DateTimeOffset.UtcNow) + public override async Task> BaseGetAsync(string cacheKey) + { + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) return CacheValue.Null; + + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + return new CacheValue(t, true); + } + else + { + return CacheValue.NoValue; + } + } + + public override IDictionary> BaseGetByPrefix(string prefix) + { + IDictionary> dict = new Dictionary>(); + + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); + + if (list == null || !list.Any()) return dict; + + foreach (var item in list) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) { - var t = MessagePackSerializer.NonGeneric.Deserialize(type, cached.Value); - return t; + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.Null); + } } else { - return null; + var cached = GetDiskCacheValue(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } } } + + return dict; } - public override async Task> BaseGetAsync(string cacheKey) + public override async Task>> BaseGetByPrefixAsync(string prefix) { - var path = GetFilePath(cacheKey); + IDictionary> dict = new Dictionary>(); - if (!File.Exists(path)) return CacheValue.Null; + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + if (list == null || !list.Any()) return dict; + + foreach (var item in list) { - var cached = await MessagePackSerializer.DeserializeAsync(stream); + var path = GetRawPath(item); - if (cached.Expiration > DateTimeOffset.UtcNow) + if (!File.Exists(path)) { - var t = MessagePackSerializer.Deserialize(cached.Value); - return new CacheValue(t, true); + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.Null); + } } else { - return CacheValue.NoValue; + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } } } - } - - public override IDictionary> BaseGetByPrefix(string prefix) - { - throw new NotImplementedException(); - } - public override Task>> BaseGetByPrefixAsync(string prefix) - { - throw new NotImplementedException(); + return dict; } public override int BaseGetCount(string prefix = "") @@ -301,36 +400,30 @@ public override int BaseGetCount(string prefix = "") public override TimeSpan BaseGetExpiration(string cacheKey) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) { return TimeSpan.Zero; } - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var cached = MessagePackSerializer.Deserialize(stream); + var cached = GetDiskCacheValue(path); - return cached.Expiration.Subtract(DateTimeOffset.UtcNow); - } + return cached.Expiration.Subtract(DateTimeOffset.UtcNow); } public override async Task BaseGetExpirationAsync(string cacheKey) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) { return TimeSpan.Zero; } - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var cached = await MessagePackSerializer.DeserializeAsync(stream); + var cached = await GetDiskCacheValueAsync(path); - return cached.Expiration.Subtract(DateTimeOffset.UtcNow); - } + return cached.Expiration.Subtract(DateTimeOffset.UtcNow); } public override void BaseRefresh(string cacheKey, T cacheValue, TimeSpan expiration) @@ -346,7 +439,7 @@ public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan public override void BaseRemove(string cacheKey) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) { @@ -354,38 +447,24 @@ public override void BaseRemove(string cacheKey) //return true; } - try - { - File.Delete(path); - //return true; - } - catch - { - //return false; - } + DeleteFileWithRetry(path); } public override void BaseRemoveAll(IEnumerable cacheKeys) - { + { foreach (string key in cacheKeys) { if (string.IsNullOrWhiteSpace(key)) continue; - var path = GetFilePath(key); + var path = GetRawPath(key); if (!File.Exists(path)) { continue; } - try - { - File.Delete(path); - } - catch - { - } + DeleteFileWithRetry(path); } } @@ -396,20 +475,14 @@ public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) if (string.IsNullOrWhiteSpace(key)) continue; - var path = GetFilePath(key); + var path = GetRawPath(key); if (!File.Exists(path)) { continue; } - try - { - File.Delete(path); - } - catch - { - } + DeleteFileWithRetry(path); } return Task.CompletedTask; @@ -417,7 +490,7 @@ public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) public override Task BaseRemoveAsync(string cacheKey) { - var path = GetFilePath(cacheKey); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) { @@ -425,44 +498,50 @@ public override Task BaseRemoveAsync(string cacheKey) //return true; } - try - { - File.Delete(path); - //return true; - } - catch - { - //return false; - } + DeleteFileWithRetry(path); return Task.CompletedTask; } public override void BaseRemoveByPrefix(string prefix) { - throw new NotImplementedException(); + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Value).ToList(); + + foreach (var item in list) + { + var path = BuildRawPath(item); + + DeleteFileWithRetry(path); + } } public override Task BaseRemoveByPrefixAsync(string prefix) { - throw new NotImplementedException(); + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Value).ToList(); + + foreach (var item in list) + { + var path = BuildRawPath(item); + + DeleteFileWithRetry(path); + } + + return Task.CompletedTask; } public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expiration) { - var path = GetFilePath(cacheKey); + var (path, fileName) = GetFilePath(cacheKey); - var value = MessagePackSerializer.Serialize(cacheValue); - - var cached = new DiskCacheValue(value, (int)expiration.TotalSeconds); - - var bytes = MessagePackSerializer.Serialize(cached); + var bytes = BuildDiskCacheValue(cacheValue, expiration); using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) { stream.Write(bytes, 0, bytes.Length); //return true; } + + AppendKey(cacheKey, fileName); } public override void BaseSetAll(IDictionary values, TimeSpan expiration) @@ -471,18 +550,16 @@ public override void BaseSetAll(IDictionary values, TimeSpan expir { try { - var path = GetFilePath(item.Key); + var (path, fileName) = GetFilePath(item.Key); - var val = MessagePackSerializer.Serialize(item.Value); - - var cached = new DiskCacheValue(val, (int)expiration.TotalSeconds); - - var bytes = MessagePackSerializer.Serialize(cached); + var bytes = BuildDiskCacheValue(item.Value, expiration); using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) { stream.Write(bytes, 0, bytes.Length); } + + AppendKey(item.Key, fileName); } catch { @@ -497,18 +574,16 @@ public override async Task BaseSetAllAsync(IDictionary values, Tim { try { - var path = GetFilePath(item.Key); - - var val = MessagePackSerializer.Serialize(item.Value); - - var cached = new DiskCacheValue(val, (int)expiration.TotalSeconds); + var (path, fileName) = GetFilePath(item.Key); - var bytes = MessagePackSerializer.Serialize(cached); + var bytes = BuildDiskCacheValue(item.Value, expiration); using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) { await stream.WriteAsync(bytes, 0, bytes.Length); } + + AppendKey(item.Key, fileName); } catch { @@ -519,36 +594,194 @@ public override async Task BaseSetAllAsync(IDictionary values, Tim public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) { - var path = GetFilePath(cacheKey); - - var value = MessagePackSerializer.Serialize(cacheValue); + var (path, fileName) = GetFilePath(cacheKey); - var cached = new DiskCacheValue(value, (int)expiration.TotalSeconds); - - var bytes = MessagePackSerializer.Serialize(cached); + var bytes = BuildDiskCacheValue(cacheValue, expiration); using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) { await stream.WriteAsync(bytes, 0, bytes.Length); //return true; } + + AppendKey(cacheKey, fileName); } public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expiration) { - throw new NotImplementedException(); + var (path, fileName) = GetFilePath(cacheKey); + + if (File.Exists(path)) + { + var cached = GetDiskCacheValue(path); + + if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) + { + return true; + } + } + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + stream.Write(bytes, 0, bytes.Length); + AppendKey(cacheKey, fileName); + return true; + } } - public override Task BaseTrySetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + public override async Task BaseTrySetAsync(string cacheKey, T cacheValue, TimeSpan expiration) { - throw new NotImplementedException(); + var (path, fileName) = GetFilePath(cacheKey); + + if (File.Exists(path)) + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) + { + return true; + } + } + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + AppendKey(cacheKey, fileName); + return true; + } } - private string GetFilePath(string key) + private (string path, string md5Name) GetFilePath(string key) { - // TODO: Special characters for file name - var path = Path.Combine(_options.DBConfig.BasePath, _name, $"{key}.dat"); + var md5FolderName = GetMd5Str(_name); + var md5FileName = GetMd5Str(key); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"{md5FileName}.dat"); + return (path, md5FileName); + } + + private string BuildRawPath(string key) + { + var md5FolderName = GetMd5Str(_name); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"{key}.dat"); return path; } + + private string GetRawPath(string cacheKey) + { + var path = string.Empty; + if (_cacheKeysMap.TryGetValue(cacheKey, out var fileName)) + { + path = BuildRawPath(fileName); + } + else + { + path = BuildMd5Path(cacheKey); + } + return path; + } + + + private string BuildMd5Path(string key) + { + var md5FolderName = GetMd5Str(_name); + var md5FileName = GetMd5Str(key); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"{md5FileName}.dat"); + return path; + } + + private void DeleteFileWithRetry(string path) + { + for (int i = 0; i < 3; i++) + { + try + { + File.Delete(path); + break; + } + catch + { + //return false; + } + } + } + + private DiskCacheValue GetDiskCacheValue(string path) + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = MessagePackSerializer.Deserialize(stream); + + return cached; + } + } + + private void AppendKey(string key, string md5Key) + { + if (_cacheKeysMap.TryAdd(key, md5Key)) + { + var md5FolderName = GetMd5Str(_name); + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"key.dat"); + + var bytes = Encoding.UTF8.GetBytes(key); + + for (int i = 0; i < 3; i++) + { + try + { + using (FileStream stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read)) + { + stream.Write(bytes, 0, bytes.Length); + } + + break; + } + catch + { + + } + } + } + } + + private async Task GetDiskCacheValueAsync(string path) + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = await MessagePackSerializer.DeserializeAsync(stream); + + return cached; + } + } + + private byte[] BuildDiskCacheValue(T t, TimeSpan ts) + { + var value = MessagePackSerializer.Serialize(t); + + var cached = new DiskCacheValue(value, (int)ts.TotalSeconds); + + var bytes = MessagePackSerializer.Serialize(cached); + + return bytes; + } + + private string GetMd5Str(string src) + { + using (MD5 md5 = MD5.Create()) + { + byte[] bytes_md5_in = Encoding.UTF8.GetBytes(src); + byte[] bytes_md5_out = md5.ComputeHash(bytes_md5_in); + + var md5_out = BitConverter.ToString(bytes_md5_out); + + return md5_out; + } + } } } From bbc757614ca8ed83b9e3b8993e0b29ffdbcc972e Mon Sep 17 00:00:00 2001 From: catcherwong Date: Mon, 24 Jun 2019 06:46:51 +0800 Subject: [PATCH 06/12] :construction: Add SaveKeyTimer --- .../DefaultDiskCachingProvider.cs | 188 ++++++++++++++---- 1 file changed, 147 insertions(+), 41 deletions(-) diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index b6ccae86..7f1f79c8 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Security.Cryptography; using System.Text; + using System.Threading; using System.Threading.Tasks; using EasyCaching.Core; using MessagePack; @@ -36,6 +37,7 @@ public class DefaultDiskCachingProvider : EasyCachingAbstractProvider private readonly ConcurrentDictionary _cacheKeysMap; + private Timer _saveKeyTimer; public DefaultDiskCachingProvider(string name, DiskOptions options, @@ -56,6 +58,41 @@ public DefaultDiskCachingProvider(string name, this.IsDistributedProvider = false; InitCacheKey(); + + _saveKeyTimer = new Timer(SaveKeyToFile, null, TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(2)); + } + + private void SaveKeyToFile(object state) + { + if (!_cacheKeysMap.IsEmpty) + { + var md5FolderName = GetMd5Str(_name); + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"key.dat"); + + var keys = _cacheKeysMap.Keys.ToArray(); + + var value = string.Join("\n", keys); + + var bytes = Encoding.UTF8.GetBytes(value); + + for (int i = 0; i < 3; i++) + { + try + { + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write)) + { + // batch , not all in one + stream.Write(bytes, 0, bytes.Length); + } + + break; + } + catch + { + + } + } + } } private void InitCacheKey() @@ -78,11 +115,11 @@ private void InitCacheKey() _cacheKeysMap.TryAdd(line, GetMd5Str(line)); } } - } + } break; } - catch + catch { } @@ -165,14 +202,14 @@ public override CacheValue BaseGet(string cacheKey, Func dataRetriever, if (res != null) { Set(cacheKey, res, expiration); - //remove mutex key + // remove mutex key _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); return new CacheValue(res, true); } else { - //remove mutex key + // remove mutex key _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); return CacheValue.NoValue; } @@ -199,12 +236,84 @@ public override CacheValue BaseGet(string cacheKey) public override IDictionary> BaseGetAll(IEnumerable cacheKeys) { - throw new NotImplementedException(); + IDictionary> dict = new Dictionary>(); + + foreach (var item in cacheKeys) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.Null); + } + } + else + { + var cached = GetDiskCacheValue(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; } - public override Task>> BaseGetAllAsync(IEnumerable cacheKeys) + public override async Task>> BaseGetAllAsync(IEnumerable cacheKeys) { - throw new NotImplementedException(); + IDictionary> dict = new Dictionary>(); + + foreach (var item in cacheKeys) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.Null); + } + } + else + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; } public override async Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) @@ -464,7 +573,10 @@ public override void BaseRemoveAll(IEnumerable cacheKeys) continue; } - DeleteFileWithRetry(path); + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(key, out _); + } } } @@ -482,7 +594,10 @@ public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) continue; } - DeleteFileWithRetry(path); + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(key, out _); + } } return Task.CompletedTask; @@ -498,32 +613,41 @@ public override Task BaseRemoveAsync(string cacheKey) //return true; } - DeleteFileWithRetry(path); + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(cacheKey, out _); + } return Task.CompletedTask; } public override void BaseRemoveByPrefix(string prefix) { - var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Value).ToList(); + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); foreach (var item in list) { - var path = BuildRawPath(item); + var path = BuildMd5Path(item); - DeleteFileWithRetry(path); + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(item, out _); + } } } public override Task BaseRemoveByPrefixAsync(string prefix) { - var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Value).ToList(); + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); foreach (var item in list) { - var path = BuildRawPath(item); + var path = BuildMd5Path(item); - DeleteFileWithRetry(path); + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(item, out _); + } } return Task.CompletedTask; @@ -696,13 +820,16 @@ private string BuildMd5Path(string key) return path; } - private void DeleteFileWithRetry(string path) + private bool DeleteFileWithRetry(string path) { + bool flag = false; + for (int i = 0; i < 3; i++) { try { File.Delete(path); + flag = true; break; } catch @@ -710,6 +837,8 @@ private void DeleteFileWithRetry(string path) //return false; } } + + return flag; } private DiskCacheValue GetDiskCacheValue(string path) @@ -724,30 +853,7 @@ private DiskCacheValue GetDiskCacheValue(string path) private void AppendKey(string key, string md5Key) { - if (_cacheKeysMap.TryAdd(key, md5Key)) - { - var md5FolderName = GetMd5Str(_name); - var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"key.dat"); - - var bytes = Encoding.UTF8.GetBytes(key); - - for (int i = 0; i < 3; i++) - { - try - { - using (FileStream stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read)) - { - stream.Write(bytes, 0, bytes.Length); - } - - break; - } - catch - { - - } - } - } + _cacheKeysMap.TryAdd(key, md5Key); } private async Task GetDiskCacheValueAsync(string path) From 5ac21f48ad353f41a85ecd1a4804eb49e7f53d3a Mon Sep 17 00:00:00 2001 From: catcherwong Date: Sat, 29 Jun 2019 23:02:15 +0800 Subject: [PATCH 07/12] :construction: Fixed some issues --- .../Internal/EasyCachingConstValue.cs | 10 ++ .../Configurations/DiskOptionsExtension.cs | 43 ++++++- .../EasyCachingOptionsExtensions.cs | 37 ++++-- .../DefaultDiskCachingProvider.cs | 115 ++++++++++++++++-- .../Internal/DiskCacheValue.cs | 5 +- .../CachingTests/BaseCachingProviderTest.cs | 8 +- .../CachingTests/DiskCachingProviderTest.cs | 54 ++++++++ .../EasyCaching.UnitTests.csproj | 1 + 8 files changed, 247 insertions(+), 26 deletions(-) create mode 100644 test/EasyCaching.UnitTests/CachingTests/DiskCachingProviderTest.cs diff --git a/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs b/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs index e6b95018..89c05689 100644 --- a/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs +++ b/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs @@ -35,6 +35,11 @@ public class EasyCachingConstValue /// public const string InMemorySection = "easycaching:inmemory"; + /// + /// The disk section. + /// + public const string DiskSection = "easycaching:disk"; + /// /// The redis bus section. /// @@ -70,6 +75,11 @@ public class EasyCachingConstValue /// public const string DefaultSQLiteName = "DefaultSQLite"; + /// + /// The default name of the disk. + /// + public const string DefaultDiskName = "DefaultDisk"; + /// /// The default name of the serializer. /// diff --git a/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs b/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs index a8129a16..7f59f49a 100644 --- a/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs +++ b/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs @@ -1,12 +1,49 @@ namespace EasyCaching.Disk { + using System; + using EasyCaching.Core; using EasyCaching.Core.Configurations; + using MessagePack.Resolvers; + using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.DependencyInjection.Extensions; - public static class DiskOptionsExtension + internal sealed class DiskOptionsExtension : IEasyCachingOptionsExtension { - public static EasyCachingOptions UseDisk(this EasyCachingOptions options, string name = "") + /// + /// The name. + /// + private readonly string _name; + /// + /// The configure. + /// + private readonly Action configure; + + public DiskOptionsExtension(string name, Action configure) + { + this._name = name; + this.configure = configure; + } + + public void AddServices(IServiceCollection services) + { + services.AddOptions(); + services.Configure(_name, configure); + + services.TryAddSingleton(); + services.AddSingleton(x => + { + var optionsMon = x.GetRequiredService>(); + var options = optionsMon.Get(_name); + //ILoggerFactory can be null + var factory = x.GetService(); + return new DefaultDiskCachingProvider(_name, options, factory); + }); + } + + public void WithServices(IApplicationBuilder app) { - return null; + // Method intentionally left empty. } } } diff --git a/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs b/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs index 05e53d6b..62df72dd 100644 --- a/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs +++ b/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs @@ -1,23 +1,40 @@ namespace EasyCaching.Disk { + using System; + using EasyCaching.Core; using EasyCaching.Core.Configurations; - using Microsoft.AspNetCore.Builder; - using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Configuration; - internal sealed class EasyCachingOptionsExtensions : IEasyCachingOptionsExtension + public static class EasyCachingOptionsExtensions { - public EasyCachingOptionsExtensions() + /// + /// Uses the disk caching provider. + /// + /// Options. + /// Configure. + /// Name. + public static EasyCachingOptions UseDisk(this EasyCachingOptions options, Action configure, string name = EasyCachingConstValue.DefaultDiskName) { - } + ArgumentCheck.NotNull(configure, nameof(configure)); - public void AddServices(IServiceCollection services) - { - throw new System.NotImplementedException(); + options.RegisterExtension(new DiskOptionsExtension(name, configure)); + + return options; } - public void WithServices(IApplicationBuilder app) + public static EasyCachingOptions UseDisk(this EasyCachingOptions options, IConfiguration configuration, string name = EasyCachingConstValue.DefaultDiskName, string sectionName = EasyCachingConstValue.DiskSection) { - throw new System.NotImplementedException(); + var dbConfig = configuration.GetSection(sectionName); + var diskOptions = new DiskOptions(); + dbConfig.Bind(diskOptions); + + void configure(DiskOptions x) + { + x.EnableLogging = diskOptions.EnableLogging; + x.MaxRdSecond = diskOptions.MaxRdSecond; + x.DBConfig = diskOptions.DBConfig; + } + return options.UseDisk(configure, name); } } } diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index 7f1f79c8..8ad08681 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using EasyCaching.Core; using MessagePack; + using MessagePack.Resolvers; using Microsoft.Extensions.Logging; public class DefaultDiskCachingProvider : EasyCachingAbstractProvider @@ -57,11 +58,32 @@ public DefaultDiskCachingProvider(string name, this.ProviderMaxRdSecond = _options.MaxRdSecond; this.IsDistributedProvider = false; - InitCacheKey(); + Init(); _saveKeyTimer = new Timer(SaveKeyToFile, null, TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(2)); } + private void Init() + { + var md5FolderName = GetMd5Str(_name); + + var basePath = Path.Combine(_options.DBConfig.BasePath, md5FolderName); + + if (!Directory.Exists(basePath)) + { + Directory.CreateDirectory(basePath); + } + + var path = Path.Combine(basePath, $"key.dat"); + + if (!File.Exists(path)) + { + File.Create(path); + } + + InitCacheKey(); + } + private void SaveKeyToFile(object state) { if (!_cacheKeysMap.IsEmpty) @@ -128,6 +150,8 @@ private void InitCacheKey() public override bool BaseExists(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = BuildMd5Path(cacheKey); if (!File.Exists(path)) return false; @@ -139,6 +163,8 @@ public override bool BaseExists(string cacheKey) public override async Task BaseExistsAsync(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = BuildMd5Path(cacheKey); if (!File.Exists(path)) return false; @@ -150,6 +176,9 @@ public override async Task BaseExistsAsync(string cacheKey) public override void BaseFlush() { + if (_options.EnableLogging) + _logger?.LogInformation("Flush"); + var path = Path.Combine(_options.DBConfig.BasePath, _name); Directory.Delete(path, true); @@ -157,6 +186,9 @@ public override void BaseFlush() public override Task BaseFlushAsync() { + if (_options.EnableLogging) + _logger?.LogInformation("FlushAsync"); + var path = Path.Combine(_options.DBConfig.BasePath, _name); Directory.Delete(path, true); @@ -166,6 +198,9 @@ public override Task BaseFlushAsync() public override CacheValue BaseGet(string cacheKey, Func dataRetriever, TimeSpan expiration) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + var path = GetRawPath(cacheKey); if (File.Exists(path)) @@ -217,6 +252,8 @@ public override CacheValue BaseGet(string cacheKey, Func dataRetriever, public override CacheValue BaseGet(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) return CacheValue.Null; @@ -236,6 +273,8 @@ public override CacheValue BaseGet(string cacheKey) public override IDictionary> BaseGetAll(IEnumerable cacheKeys) { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + IDictionary> dict = new Dictionary>(); foreach (var item in cacheKeys) @@ -277,6 +316,8 @@ public override IDictionary> BaseGetAll(IEnumerable>> BaseGetAllAsync(IEnumerable cacheKeys) { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + IDictionary> dict = new Dictionary>(); foreach (var item in cacheKeys) @@ -318,6 +359,9 @@ public override async Task>> BaseGetAllAsync> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + var path = GetRawPath(cacheKey); if (File.Exists(path)) @@ -368,6 +412,8 @@ public override async Task> BaseGetAsync(string cacheKey, Func< public override async Task BaseGetAsync(string cacheKey, Type type) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) @@ -395,6 +441,8 @@ public override async Task BaseGetAsync(string cacheKey, Type type) public override async Task> BaseGetAsync(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) return CacheValue.Null; @@ -414,6 +462,8 @@ public override async Task> BaseGetAsync(string cacheKey) public override IDictionary> BaseGetByPrefix(string prefix) { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + IDictionary> dict = new Dictionary>(); var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); @@ -459,6 +509,8 @@ public override IDictionary> BaseGetByPrefix(string pre public override async Task>> BaseGetByPrefixAsync(string prefix) { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + IDictionary> dict = new Dictionary>(); var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); @@ -504,11 +556,20 @@ public override async Task>> BaseGetByPrefixAs public override int BaseGetCount(string prefix = "") { - throw new NotImplementedException(); + if (string.IsNullOrWhiteSpace(prefix)) + { + return _cacheKeysMap.Count; + } + else + { + return _cacheKeysMap.Count(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)); + } } public override TimeSpan BaseGetExpiration(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) @@ -523,6 +584,8 @@ public override TimeSpan BaseGetExpiration(string cacheKey) public override async Task BaseGetExpirationAsync(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) @@ -548,6 +611,8 @@ public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan public override void BaseRemove(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) @@ -561,6 +626,8 @@ public override void BaseRemove(string cacheKey) public override void BaseRemoveAll(IEnumerable cacheKeys) { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + foreach (string key in cacheKeys) { if (string.IsNullOrWhiteSpace(key)) @@ -582,6 +649,8 @@ public override void BaseRemoveAll(IEnumerable cacheKeys) public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + foreach (string key in cacheKeys) { if (string.IsNullOrWhiteSpace(key)) @@ -605,6 +674,8 @@ public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) public override Task BaseRemoveAsync(string cacheKey) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + var path = GetRawPath(cacheKey); if (!File.Exists(path)) @@ -623,6 +694,8 @@ public override Task BaseRemoveAsync(string cacheKey) public override void BaseRemoveByPrefix(string prefix) { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); foreach (var item in list) @@ -638,6 +711,8 @@ public override void BaseRemoveByPrefix(string prefix) public override Task BaseRemoveByPrefixAsync(string prefix) { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); foreach (var item in list) @@ -655,6 +730,10 @@ public override Task BaseRemoveByPrefixAsync(string prefix) public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expiration) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + var (path, fileName) = GetFilePath(cacheKey); var bytes = BuildDiskCacheValue(cacheValue, expiration); @@ -670,6 +749,9 @@ public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expirati public override void BaseSetAll(IDictionary values, TimeSpan expiration) { + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + ArgumentCheck.NotNullAndCountGTZero(values, nameof(values)); + foreach (var item in values) { try @@ -694,6 +776,9 @@ public override void BaseSetAll(IDictionary values, TimeSpan expir public override async Task BaseSetAllAsync(IDictionary values, TimeSpan expiration) { + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + ArgumentCheck.NotNullAndCountGTZero(values, nameof(values)); + foreach (var item in values) { try @@ -718,6 +803,10 @@ public override async Task BaseSetAllAsync(IDictionary values, Tim public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + var (path, fileName) = GetFilePath(cacheKey); var bytes = BuildDiskCacheValue(cacheValue, expiration); @@ -733,6 +822,10 @@ public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSp public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expiration) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + var (path, fileName) = GetFilePath(cacheKey); if (File.Exists(path)) @@ -757,6 +850,10 @@ public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expir public override async Task BaseTrySetAsync(string cacheKey, T cacheValue, TimeSpan expiration) { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + var (path, fileName) = GetFilePath(cacheKey); if (File.Exists(path)) @@ -870,7 +967,7 @@ private byte[] BuildDiskCacheValue(T t, TimeSpan ts) { var value = MessagePackSerializer.Serialize(t); - var cached = new DiskCacheValue(value, (int)ts.TotalSeconds); + var cached = new DiskCacheValue(value, DateTimeOffset.UtcNow.AddSeconds((int)ts.TotalSeconds)); var bytes = MessagePackSerializer.Serialize(cached); @@ -881,12 +978,16 @@ private string GetMd5Str(string src) { using (MD5 md5 = MD5.Create()) { - byte[] bytes_md5_in = Encoding.UTF8.GetBytes(src); - byte[] bytes_md5_out = md5.ComputeHash(bytes_md5_in); + byte[] data = md5.ComputeHash(Encoding.UTF8.GetBytes(src)); - var md5_out = BitConverter.ToString(bytes_md5_out); + StringBuilder sBuilder = new StringBuilder(64); + + for (int i = 0; i < data.Length; i++) + { + sBuilder.Append(data[i].ToString("x2")); + } - return md5_out; + return sBuilder.ToString(); } } } diff --git a/src/EasyCaching.Disk/Internal/DiskCacheValue.cs b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs index 4cfdeaa0..de5e45aa 100644 --- a/src/EasyCaching.Disk/Internal/DiskCacheValue.cs +++ b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs @@ -3,13 +3,14 @@ using System; using MessagePack; + [MessagePackObject] public class DiskCacheValue { [SerializationConstructor] - public DiskCacheValue(byte[] val, int second) + public DiskCacheValue(byte[] val, DateTimeOffset time) { Value = val; - Expiration = DateTimeOffset.UtcNow.AddSeconds(second); + Expiration = time; } diff --git a/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs b/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs index 730c78c8..624203e0 100644 --- a/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs +++ b/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs @@ -609,7 +609,7 @@ protected virtual async Task GetAsync_Parallel_Should_Succeed() #region Refresh/RefreshAsync [Fact] - public void Refresh_Should_Succeed() + protected virtual void Refresh_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = "value"; @@ -629,7 +629,7 @@ public void Refresh_Should_Succeed() } [Fact] - public async Task Refresh_Async_Should_Succeed() + protected virtual async Task Refresh_Async_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = "value"; @@ -651,7 +651,7 @@ public async Task Refresh_Async_Should_Succeed() } [Fact] - public void Refresh_Value_Type_Object_Should_Succeed() + protected virtual void Refresh_Value_Type_Object_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = 100; @@ -671,7 +671,7 @@ public void Refresh_Value_Type_Object_Should_Succeed() } [Fact] - public async Task Refresh_Value_Type_Object_Async_Should_Succeed() + protected virtual async Task Refresh_Value_Type_Object_Async_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = 100; diff --git a/test/EasyCaching.UnitTests/CachingTests/DiskCachingProviderTest.cs b/test/EasyCaching.UnitTests/CachingTests/DiskCachingProviderTest.cs new file mode 100644 index 00000000..952acb0d --- /dev/null +++ b/test/EasyCaching.UnitTests/CachingTests/DiskCachingProviderTest.cs @@ -0,0 +1,54 @@ +namespace EasyCaching.UnitTests +{ + using System; + using System.IO; + using System.Threading.Tasks; + using EasyCaching.Core; + using EasyCaching.Disk; + using Microsoft.Extensions.DependencyInjection; + using Xunit; + + public class DiskCachingProviderTest : BaseCachingProviderTest + { + public DiskCachingProviderTest() + { + IServiceCollection services = new ServiceCollection(); + services.AddEasyCaching(x => x.UseDisk(options => + { + options.MaxRdSecond = 0; + options.DBConfig = new DiskDbOptions + { + BasePath = Path.GetTempPath() + }; + + })); + IServiceProvider serviceProvider = services.BuildServiceProvider(); + _provider = serviceProvider.GetService(); + _defaultTs = TimeSpan.FromSeconds(30); + } + + [Fact(Skip = "")] + protected override void Refresh_Should_Succeed() + { + + } + + [Fact(Skip = "")] + protected override Task Refresh_Async_Should_Succeed() + { + return Task.CompletedTask; + } + + [Fact(Skip = "")] + protected override void Refresh_Value_Type_Object_Should_Succeed() + { + + } + + [Fact(Skip = "")] + protected override Task Refresh_Value_Type_Object_Async_Should_Succeed() + { + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj b/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj index 96bac279..8d8a1367 100644 --- a/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj +++ b/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj @@ -32,6 +32,7 @@ + From 06e5af9e9ad5a41412ab37fe1f8b13a832fcf772 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Sun, 30 Jun 2019 10:31:11 +0800 Subject: [PATCH 08/12] :construction: Object Value Use ContractlessStandardResolver --- .../DefaultDiskCachingProvider.cs | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index 8ad08681..b324d978 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -11,7 +11,6 @@ using System.Threading.Tasks; using EasyCaching.Core; using MessagePack; - using MessagePack.Resolvers; using Microsoft.Extensions.Logging; public class DefaultDiskCachingProvider : EasyCachingAbstractProvider @@ -179,7 +178,9 @@ public override void BaseFlush() if (_options.EnableLogging) _logger?.LogInformation("Flush"); - var path = Path.Combine(_options.DBConfig.BasePath, _name); + var md5FolderName = GetMd5Str(_name); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); Directory.Delete(path, true); } @@ -189,7 +190,9 @@ public override Task BaseFlushAsync() if (_options.EnableLogging) _logger?.LogInformation("FlushAsync"); - var path = Path.Combine(_options.DBConfig.BasePath, _name); + var md5FolderName = GetMd5Str(_name); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); Directory.Delete(path, true); @@ -209,7 +212,7 @@ public override CacheValue BaseGet(string cacheKey, Func dataRetriever, if (cached.Expiration > DateTimeOffset.UtcNow) { - var t = MessagePackSerializer.Deserialize(cached.Value); + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); if (_options.EnableLogging) _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); @@ -262,11 +265,21 @@ public override CacheValue BaseGet(string cacheKey) if (cached.Expiration > DateTimeOffset.UtcNow) { - var t = MessagePackSerializer.Deserialize(cached.Value); + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); return new CacheValue(t, true); } else { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + return CacheValue.NoValue; } } @@ -337,7 +350,7 @@ public override async Task>> BaseGetAllAsync DateTimeOffset.UtcNow) { - var t = MessagePackSerializer.Deserialize(cached.Value); + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); if (!dict.ContainsKey(item)) { @@ -370,7 +383,7 @@ public override async Task> BaseGetAsync(string cacheKey, Func< if (cached.Expiration > DateTimeOffset.UtcNow) { - var t = MessagePackSerializer.Deserialize(cached.Value); + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); if (_options.EnableLogging) _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); @@ -430,11 +443,21 @@ public override async Task BaseGetAsync(string cacheKey, Type type) if (cached.Expiration > DateTimeOffset.UtcNow) { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + var t = MessagePackSerializer.NonGeneric.Deserialize(type, cached.Value); return t; } else { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + return null; } } @@ -451,7 +474,7 @@ public override async Task> BaseGetAsync(string cacheKey) if (cached.Expiration > DateTimeOffset.UtcNow) { - var t = MessagePackSerializer.Deserialize(cached.Value); + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); return new CacheValue(t, true); } else @@ -534,7 +557,7 @@ public override async Task>> BaseGetByPrefixAs if (cached.Expiration > DateTimeOffset.UtcNow) { - var t = MessagePackSerializer.Deserialize(cached.Value); + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); if (!dict.ContainsKey(item)) { @@ -965,7 +988,7 @@ private async Task GetDiskCacheValueAsync(string path) private byte[] BuildDiskCacheValue(T t, TimeSpan ts) { - var value = MessagePackSerializer.Serialize(t); + var value = MessagePackSerializer.Serialize(t, MessagePack.Resolvers.ContractlessStandardResolver.Instance); var cached = new DiskCacheValue(value, DateTimeOffset.UtcNow.AddSeconds((int)ts.TotalSeconds)); From ded91d538ba6f944069bb591773341a8eac28fbb Mon Sep 17 00:00:00 2001 From: catcherwong Date: Sun, 30 Jun 2019 10:36:55 +0800 Subject: [PATCH 09/12] :construction: Fixed CacheStats --- .../DefaultDiskCachingProvider.cs | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index b324d978..90a578de 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -259,7 +259,15 @@ public override CacheValue BaseGet(string cacheKey) var path = GetRawPath(cacheKey); - if (!File.Exists(path)) return CacheValue.Null; + if (!File.Exists(path)) + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + + return CacheValue.Null; + } var cached = GetDiskCacheValue(path); @@ -468,17 +476,35 @@ public override async Task> BaseGetAsync(string cacheKey) var path = GetRawPath(cacheKey); - if (!File.Exists(path)) return CacheValue.Null; + if (!File.Exists(path)) + { + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + return CacheValue.Null; + } var cached = await GetDiskCacheValueAsync(path); if (cached.Expiration > DateTimeOffset.UtcNow) { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); return new CacheValue(t, true); } else { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + return CacheValue.NoValue; } } From 4af46dfb44554f640986ebf80f5eba67caa506a6 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Mon, 1 Jul 2019 06:52:52 +0800 Subject: [PATCH 10/12] :green_heart: Fixed some bugs due to CI failed --- .../DefaultDiskCachingProvider.cs | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index 90a578de..b9784700 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -182,7 +182,9 @@ public override void BaseFlush() var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); - Directory.Delete(path, true); + DeleteDirectory(path); + + _cacheKeysMap.Clear(); } public override Task BaseFlushAsync() @@ -194,7 +196,9 @@ public override Task BaseFlushAsync() var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); - Directory.Delete(path, true); + DeleteDirectory(path); + + _cacheKeysMap.Clear(); return Task.CompletedTask; } @@ -266,7 +270,7 @@ public override CacheValue BaseGet(string cacheKey) CacheStats.OnMiss(); - return CacheValue.Null; + return CacheValue.NoValue; } var cached = GetDiskCacheValue(path); @@ -306,7 +310,7 @@ public override IDictionary> BaseGetAll(IEnumerable.Null); + dict.Add(item, CacheValue.NoValue); } } else @@ -349,7 +353,7 @@ public override async Task>> BaseGetAllAsync.Null); + dict.Add(item, CacheValue.NoValue); } } else @@ -482,8 +486,8 @@ public override async Task> BaseGetAsync(string cacheKey) if (_options.EnableLogging) _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); - - return CacheValue.Null; + + return CacheValue.NoValue; } var cached = await GetDiskCacheValueAsync(path); @@ -527,7 +531,7 @@ public override IDictionary> BaseGetByPrefix(string pre { if (!dict.ContainsKey(item)) { - dict.Add(item, CacheValue.Null); + dict.Add(item, CacheValue.NoValue); } } else @@ -574,7 +578,7 @@ public override async Task>> BaseGetByPrefixAs { if (!dict.ContainsKey(item)) { - dict.Add(item, CacheValue.Null); + dict.Add(item, CacheValue.NoValue); } } else @@ -670,7 +674,10 @@ public override void BaseRemove(string cacheKey) //return true; } - DeleteFileWithRetry(path); + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(cacheKey, out _); + } } public override void BaseRemoveAll(IEnumerable cacheKeys) @@ -787,7 +794,7 @@ public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expirati var bytes = BuildDiskCacheValue(cacheValue, expiration); - using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { stream.Write(bytes, 0, bytes.Length); //return true; @@ -809,7 +816,7 @@ public override void BaseSetAll(IDictionary values, TimeSpan expir var bytes = BuildDiskCacheValue(item.Value, expiration); - using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { stream.Write(bytes, 0, bytes.Length); } @@ -836,7 +843,7 @@ public override async Task BaseSetAllAsync(IDictionary values, Tim var bytes = BuildDiskCacheValue(item.Value, expiration); - using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { await stream.WriteAsync(bytes, 0, bytes.Length); } @@ -860,7 +867,7 @@ public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSp var bytes = BuildDiskCacheValue(cacheValue, expiration); - using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { await stream.WriteAsync(bytes, 0, bytes.Length); //return true; @@ -883,7 +890,7 @@ public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expir if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) { - return true; + return false; } } @@ -911,7 +918,7 @@ public override async Task BaseTrySetAsync(string cacheKey, T cacheValu if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) { - return true; + return false; } } @@ -1039,5 +1046,30 @@ private string GetMd5Str(string src) return sBuilder.ToString(); } } + + private void DeleteDirectory(string path) + { + try + { + DirectoryInfo dir = new DirectoryInfo(path); + FileSystemInfo[] fileinfo = dir.GetFileSystemInfos(); + foreach (FileSystemInfo i in fileinfo) + { + if (i is DirectoryInfo) + { + DirectoryInfo subdir = new DirectoryInfo(i.FullName); + subdir.Delete(true); + } + else + { + File.Delete(i.FullName); + } + } + } + catch + { + + } + } } } From c858495367597951df9b5eefccec4066808ab363 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Mon, 1 Jul 2019 06:55:06 +0800 Subject: [PATCH 11/12] :wheelchair: Add a new CachingProviderType for disk --- src/EasyCaching.Core/Internal/CachingProviderType.cs | 1 + src/EasyCaching.Disk/DefaultDiskCachingProvider.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/EasyCaching.Core/Internal/CachingProviderType.cs b/src/EasyCaching.Core/Internal/CachingProviderType.cs index d8e198c4..2103f00f 100644 --- a/src/EasyCaching.Core/Internal/CachingProviderType.cs +++ b/src/EasyCaching.Core/Internal/CachingProviderType.cs @@ -9,6 +9,7 @@ public enum CachingProviderType Memcached, Redis, SQLite, + Disk, Ext1, Ext2 } diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index b9784700..5b37e72b 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -52,7 +52,7 @@ public DefaultDiskCachingProvider(string name, this._cacheStats = new CacheStats(); this.ProviderName = _name; - this.ProviderType = CachingProviderType.Ext1; + this.ProviderType = CachingProviderType.Disk; this.ProviderStats = _cacheStats; this.ProviderMaxRdSecond = _options.MaxRdSecond; this.IsDistributedProvider = false; From 2524c73449413b8ad7077a20e18af697365658f4 Mon Sep 17 00:00:00 2001 From: catcherwong Date: Tue, 2 Jul 2019 06:41:01 +0800 Subject: [PATCH 12/12] :art: Separation of Sync and Async methods --- .../DefaultDiskCachingProvider.Async.cs | 434 +++++++++++++++++ .../DefaultDiskCachingProvider.cs | 435 +----------------- 2 files changed, 441 insertions(+), 428 deletions(-) create mode 100644 src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs new file mode 100644 index 00000000..8b23d6a3 --- /dev/null +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs @@ -0,0 +1,434 @@ +namespace EasyCaching.Disk +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using EasyCaching.Core; + using MessagePack; + using Microsoft.Extensions.Logging; + + public partial class DefaultDiskCachingProvider : EasyCachingAbstractProvider + { + public override async Task BaseExistsAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = BuildMd5Path(cacheKey); + + if (!File.Exists(path)) return false; + + var val = await GetDiskCacheValueAsync(path); + + return val.Expiration > DateTimeOffset.UtcNow; + } + + public override Task BaseFlushAsync() + { + if (_options.EnableLogging) + _logger?.LogInformation("FlushAsync"); + + var md5FolderName = GetMd5Str(_name); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); + + DeleteDirectory(path); + + _cacheKeysMap.Clear(); + + return Task.CompletedTask; + } + + public override async Task>> BaseGetAllAsync(IEnumerable cacheKeys) + { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + + IDictionary> dict = new Dictionary>(); + + foreach (var item in cacheKeys) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + else + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; + } + + public override async Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var path = GetRawPath(cacheKey); + + if (File.Exists(path)) + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + return new CacheValue(t, true); + } + } + + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + // TODO: how to add mutex key here + if (!_cacheKeysMap.TryAdd($"{cacheKey}_Lock", "1")) + { + System.Threading.Thread.Sleep(_options.SleepMs); + return await GetAsync(cacheKey, dataRetriever, expiration); + } + + var res = await dataRetriever(); + + if (res != null) + { + await SetAsync(cacheKey, res, expiration); + //remove mutex key + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); + return new CacheValue(res, true); + } + else + { + //remove mutex key + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); + return CacheValue.NoValue; + } + } + + public override async Task BaseGetAsync(string cacheKey, Type type) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + return null; + } + + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + var t = MessagePackSerializer.NonGeneric.Deserialize(type, cached.Value); + return t; + } + else + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + + return null; + } + } + + public override async Task> BaseGetAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + return CacheValue.NoValue; + } + + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + return new CacheValue(t, true); + } + else + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + + return CacheValue.NoValue; + } + } + + public override async Task>> BaseGetByPrefixAsync(string prefix) + { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + + IDictionary> dict = new Dictionary>(); + + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); + + if (list == null || !list.Any()) return dict; + + foreach (var item in list) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + else + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; + } + + public override async Task BaseGetExpirationAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + return TimeSpan.Zero; + } + + var cached = await GetDiskCacheValueAsync(path); + + return cached.Expiration.Subtract(DateTimeOffset.UtcNow); + } + + public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + // Obsolete + return Task.CompletedTask; + } + + public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) + { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + + foreach (string key in cacheKeys) + { + if (string.IsNullOrWhiteSpace(key)) + continue; + + var path = GetRawPath(key); + + if (!File.Exists(path)) + { + continue; + } + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(key, out _); + } + } + + return Task.CompletedTask; + } + + public override Task BaseRemoveAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + return Task.CompletedTask; + //return true; + } + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(cacheKey, out _); + } + + return Task.CompletedTask; + } + + public override Task BaseRemoveByPrefixAsync(string prefix) + { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); + + foreach (var item in list) + { + var path = BuildMd5Path(item); + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(item, out _); + } + } + + return Task.CompletedTask; + } + + public override async Task BaseSetAllAsync(IDictionary values, TimeSpan expiration) + { + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + ArgumentCheck.NotNullAndCountGTZero(values, nameof(values)); + + foreach (var item in values) + { + try + { + var (path, fileName) = GetFilePath(item.Key); + + var bytes = BuildDiskCacheValue(item.Value, expiration); + + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + } + + AppendKey(item.Key, fileName); + } + catch + { + + } + } + } + + public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var (path, fileName) = GetFilePath(cacheKey); + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + //return true; + } + + AppendKey(cacheKey, fileName); + } + + public override async Task BaseTrySetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var (path, fileName) = GetFilePath(cacheKey); + + if (File.Exists(path)) + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) + { + return false; + } + } + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + AppendKey(cacheKey, fileName); + return true; + } + } + + private async Task GetDiskCacheValueAsync(string path) + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = await MessagePackSerializer.DeserializeAsync(stream); + + return cached; + } + } + } +} diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs index 5b37e72b..aca88afc 100644 --- a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -8,12 +8,11 @@ using System.Security.Cryptography; using System.Text; using System.Threading; - using System.Threading.Tasks; using EasyCaching.Core; using MessagePack; using Microsoft.Extensions.Logging; - public class DefaultDiskCachingProvider : EasyCachingAbstractProvider + public partial class DefaultDiskCachingProvider : EasyCachingAbstractProvider { /// /// The options. @@ -159,20 +158,7 @@ public override bool BaseExists(string cacheKey) return val.Expiration > DateTimeOffset.UtcNow; } - - public override async Task BaseExistsAsync(string cacheKey) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - - var path = BuildMd5Path(cacheKey); - - if (!File.Exists(path)) return false; - - var val = await GetDiskCacheValueAsync(path); - - return val.Expiration > DateTimeOffset.UtcNow; - } - + public override void BaseFlush() { if (_options.EnableLogging) @@ -187,22 +173,6 @@ public override void BaseFlush() _cacheKeysMap.Clear(); } - public override Task BaseFlushAsync() - { - if (_options.EnableLogging) - _logger?.LogInformation("FlushAsync"); - - var md5FolderName = GetMd5Str(_name); - - var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); - - DeleteDirectory(path); - - _cacheKeysMap.Clear(); - - return Task.CompletedTask; - } - public override CacheValue BaseGet(string cacheKey, Func dataRetriever, TimeSpan expiration) { ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); @@ -338,181 +308,7 @@ public override IDictionary> BaseGetAll(IEnumerable>> BaseGetAllAsync(IEnumerable cacheKeys) - { - ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); - - IDictionary> dict = new Dictionary>(); - - foreach (var item in cacheKeys) - { - var path = GetRawPath(item); - - if (!File.Exists(path)) - { - if (!dict.ContainsKey(item)) - { - dict.Add(item, CacheValue.NoValue); - } - } - else - { - var cached = await GetDiskCacheValueAsync(path); - - if (cached.Expiration > DateTimeOffset.UtcNow) - { - var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); - - if (!dict.ContainsKey(item)) - { - dict.Add(item, new CacheValue(t, true)); - } - } - else - { - if (!dict.ContainsKey(item)) - { - dict.Add(item, CacheValue.NoValue); - } - } - } - } - - return dict; - } - - public override async Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); - - var path = GetRawPath(cacheKey); - - if (File.Exists(path)) - { - var cached = await GetDiskCacheValueAsync(path); - - if (cached.Expiration > DateTimeOffset.UtcNow) - { - var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); - - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); - - CacheStats.OnHit(); - - return new CacheValue(t, true); - } - } - - CacheStats.OnMiss(); - - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); - - // TODO: how to add mutex key here - if (!_cacheKeysMap.TryAdd($"{cacheKey}_Lock", "1")) - { - System.Threading.Thread.Sleep(_options.SleepMs); - return await GetAsync(cacheKey, dataRetriever, expiration); - } - - var res = await dataRetriever(); - - if (res != null) - { - await SetAsync(cacheKey, res, expiration); - //remove mutex key - _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); - return new CacheValue(res, true); - } - else - { - //remove mutex key - _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); - return CacheValue.NoValue; - } - } - - public override async Task BaseGetAsync(string cacheKey, Type type) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - - var path = GetRawPath(cacheKey); - - if (!File.Exists(path)) - { - CacheStats.OnMiss(); - - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); - - return null; - } - - var cached = await GetDiskCacheValueAsync(path); - - if (cached.Expiration > DateTimeOffset.UtcNow) - { - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); - - CacheStats.OnHit(); - - var t = MessagePackSerializer.NonGeneric.Deserialize(type, cached.Value); - return t; - } - else - { - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); - - CacheStats.OnMiss(); - - return null; - } - } - - public override async Task> BaseGetAsync(string cacheKey) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - - var path = GetRawPath(cacheKey); - - if (!File.Exists(path)) - { - CacheStats.OnMiss(); - - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); - - return CacheValue.NoValue; - } - - var cached = await GetDiskCacheValueAsync(path); - - if (cached.Expiration > DateTimeOffset.UtcNow) - { - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); - - CacheStats.OnHit(); - - var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); - return new CacheValue(t, true); - } - else - { - if (_options.EnableLogging) - _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); - - CacheStats.OnMiss(); - - return CacheValue.NoValue; - } - } - + public override IDictionary> BaseGetByPrefix(string prefix) { ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); @@ -560,53 +356,6 @@ public override IDictionary> BaseGetByPrefix(string pre return dict; } - public override async Task>> BaseGetByPrefixAsync(string prefix) - { - ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); - - IDictionary> dict = new Dictionary>(); - - var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); - - if (list == null || !list.Any()) return dict; - - foreach (var item in list) - { - var path = GetRawPath(item); - - if (!File.Exists(path)) - { - if (!dict.ContainsKey(item)) - { - dict.Add(item, CacheValue.NoValue); - } - } - else - { - var cached = await GetDiskCacheValueAsync(path); - - if (cached.Expiration > DateTimeOffset.UtcNow) - { - var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); - - if (!dict.ContainsKey(item)) - { - dict.Add(item, new CacheValue(t, true)); - } - } - else - { - if (!dict.ContainsKey(item)) - { - dict.Add(item, CacheValue.NoValue); - } - } - } - } - - return dict; - } - public override int BaseGetCount(string prefix = "") { if (string.IsNullOrWhiteSpace(prefix)) @@ -635,33 +384,11 @@ public override TimeSpan BaseGetExpiration(string cacheKey) return cached.Expiration.Subtract(DateTimeOffset.UtcNow); } - public override async Task BaseGetExpirationAsync(string cacheKey) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - - var path = GetRawPath(cacheKey); - - if (!File.Exists(path)) - { - return TimeSpan.Zero; - } - - var cached = await GetDiskCacheValueAsync(path); - - return cached.Expiration.Subtract(DateTimeOffset.UtcNow); - } - public override void BaseRefresh(string cacheKey, T cacheValue, TimeSpan expiration) { // Obsolete } - public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan expiration) - { - // Obsolete - return Task.CompletedTask; - } - public override void BaseRemove(string cacheKey) { ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); @@ -703,51 +430,6 @@ public override void BaseRemoveAll(IEnumerable cacheKeys) } } - public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) - { - ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); - - foreach (string key in cacheKeys) - { - if (string.IsNullOrWhiteSpace(key)) - continue; - - var path = GetRawPath(key); - - if (!File.Exists(path)) - { - continue; - } - - if (DeleteFileWithRetry(path)) - { - _cacheKeysMap.TryRemove(key, out _); - } - } - - return Task.CompletedTask; - } - - public override Task BaseRemoveAsync(string cacheKey) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - - var path = GetRawPath(cacheKey); - - if (!File.Exists(path)) - { - return Task.CompletedTask; - //return true; - } - - if (DeleteFileWithRetry(path)) - { - _cacheKeysMap.TryRemove(cacheKey, out _); - } - - return Task.CompletedTask; - } - public override void BaseRemoveByPrefix(string prefix) { ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); @@ -763,26 +445,7 @@ public override void BaseRemoveByPrefix(string prefix) _cacheKeysMap.TryRemove(item, out _); } } - } - - public override Task BaseRemoveByPrefixAsync(string prefix) - { - ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); - - var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); - - foreach (var item in list) - { - var path = BuildMd5Path(item); - - if (DeleteFileWithRetry(path)) - { - _cacheKeysMap.TryRemove(item, out _); - } - } - - return Task.CompletedTask; - } + } public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expiration) { @@ -829,53 +492,7 @@ public override void BaseSetAll(IDictionary values, TimeSpan expir } } } - - public override async Task BaseSetAllAsync(IDictionary values, TimeSpan expiration) - { - ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); - ArgumentCheck.NotNullAndCountGTZero(values, nameof(values)); - - foreach (var item in values) - { - try - { - var (path, fileName) = GetFilePath(item.Key); - - var bytes = BuildDiskCacheValue(item.Value, expiration); - - using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) - { - await stream.WriteAsync(bytes, 0, bytes.Length); - } - - AppendKey(item.Key, fileName); - } - catch - { - - } - } - } - - public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); - ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); - - var (path, fileName) = GetFilePath(cacheKey); - - var bytes = BuildDiskCacheValue(cacheValue, expiration); - - using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) - { - await stream.WriteAsync(bytes, 0, bytes.Length); - //return true; - } - - AppendKey(cacheKey, fileName); - } - + public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expiration) { ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); @@ -903,35 +520,7 @@ public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expir return true; } } - - public override async Task BaseTrySetAsync(string cacheKey, T cacheValue, TimeSpan expiration) - { - ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); - ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); - ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); - - var (path, fileName) = GetFilePath(cacheKey); - - if (File.Exists(path)) - { - var cached = await GetDiskCacheValueAsync(path); - - if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) - { - return false; - } - } - - var bytes = BuildDiskCacheValue(cacheValue, expiration); - - using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None)) - { - await stream.WriteAsync(bytes, 0, bytes.Length); - AppendKey(cacheKey, fileName); - return true; - } - } - + private (string path, string md5Name) GetFilePath(string key) { var md5FolderName = GetMd5Str(_name); @@ -1007,17 +596,7 @@ private DiskCacheValue GetDiskCacheValue(string path) private void AppendKey(string key, string md5Key) { _cacheKeysMap.TryAdd(key, md5Key); - } - - private async Task GetDiskCacheValueAsync(string path) - { - using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - var cached = await MessagePackSerializer.DeserializeAsync(stream); - - return cached; - } - } + } private byte[] BuildDiskCacheValue(T t, TimeSpan ts) {