diff --git a/EasyCaching.sln b/EasyCaching.sln index e8f9f0fc..8dc5aa75 100644 --- a/EasyCaching.sln +++ b/EasyCaching.sln @@ -49,6 +49,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EasyCaching.CSRedis", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EasyCaching.Bus.CSRedis", "src\EasyCaching.Bus.CSRedis\EasyCaching.Bus.CSRedis.csproj", "{861E5373-BEF6-4AA2-92C7-8F4941A079E7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EasyCaching.Disk", "src\EasyCaching.Disk\EasyCaching.Disk.csproj", "{3D48FD75-01D6-44F9-B7C3-CB6DE784F476}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -139,6 +141,10 @@ Global {861E5373-BEF6-4AA2-92C7-8F4941A079E7}.Debug|Any CPU.Build.0 = Debug|Any CPU {861E5373-BEF6-4AA2-92C7-8F4941A079E7}.Release|Any CPU.ActiveCfg = Release|Any CPU {861E5373-BEF6-4AA2-92C7-8F4941A079E7}.Release|Any CPU.Build.0 = Release|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {CE61FAA2-0233-451C-991D-4222ED61C84B} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} @@ -162,5 +168,6 @@ Global {6EBE36A2-F128-4C63-B90A-B700D8C2F2E8} = {EBB55F65-7D07-4281-8D5E-7B0CA88E1AD0} {6584761E-E51C-408F-BE51-CA0F6269589B} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} {861E5373-BEF6-4AA2-92C7-8F4941A079E7} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} + {3D48FD75-01D6-44F9-B7C3-CB6DE784F476} = {A0F5CC7E-155F-4726-8DEB-E966950B3FE9} EndGlobalSection EndGlobal diff --git a/build/releasenotes.props b/build/releasenotes.props index 2654fab8..774164da 100644 --- a/build/releasenotes.props +++ b/build/releasenotes.props @@ -45,5 +45,8 @@ 1. Upgrading dependencies. + + 1. init. + diff --git a/build/version.props b/build/version.props index 2fc070ba..6a373beb 100644 --- a/build/version.props +++ b/build/version.props @@ -16,5 +16,6 @@ 0.5.6 0.5.6 0.5.6 + 0.5.6 diff --git a/src/EasyCaching.Core/Internal/CachingProviderType.cs b/src/EasyCaching.Core/Internal/CachingProviderType.cs index d8e198c4..2103f00f 100644 --- a/src/EasyCaching.Core/Internal/CachingProviderType.cs +++ b/src/EasyCaching.Core/Internal/CachingProviderType.cs @@ -9,6 +9,7 @@ public enum CachingProviderType Memcached, Redis, SQLite, + Disk, Ext1, Ext2 } diff --git a/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs b/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs index e6b95018..89c05689 100644 --- a/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs +++ b/src/EasyCaching.Core/Internal/EasyCachingConstValue.cs @@ -35,6 +35,11 @@ public class EasyCachingConstValue /// public const string InMemorySection = "easycaching:inmemory"; + /// + /// The disk section. + /// + public const string DiskSection = "easycaching:disk"; + /// /// The redis bus section. /// @@ -70,6 +75,11 @@ public class EasyCachingConstValue /// public const string DefaultSQLiteName = "DefaultSQLite"; + /// + /// The default name of the disk. + /// + public const string DefaultDiskName = "DefaultDisk"; + /// /// The default name of the serializer. /// diff --git a/src/EasyCaching.Disk/Configurations/DiskOptions.cs b/src/EasyCaching.Disk/Configurations/DiskOptions.cs new file mode 100644 index 00000000..d37cc3c9 --- /dev/null +++ b/src/EasyCaching.Disk/Configurations/DiskOptions.cs @@ -0,0 +1,13 @@ +namespace EasyCaching.Disk +{ + using EasyCaching.Core.Configurations; + + public class DiskOptions : BaseProviderOptions + { + public DiskOptions() + { + } + + public DiskDbOptions DBConfig { get; set; } = new DiskDbOptions(); + } +} diff --git a/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs b/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs new file mode 100644 index 00000000..7f59f49a --- /dev/null +++ b/src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs @@ -0,0 +1,49 @@ +namespace EasyCaching.Disk +{ + using System; + using EasyCaching.Core; + using EasyCaching.Core.Configurations; + using MessagePack.Resolvers; + using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.DependencyInjection.Extensions; + + internal sealed class DiskOptionsExtension : IEasyCachingOptionsExtension + { + /// + /// The name. + /// + private readonly string _name; + /// + /// The configure. + /// + private readonly Action configure; + + public DiskOptionsExtension(string name, Action configure) + { + this._name = name; + this.configure = configure; + } + + public void AddServices(IServiceCollection services) + { + services.AddOptions(); + services.Configure(_name, configure); + + services.TryAddSingleton(); + services.AddSingleton(x => + { + var optionsMon = x.GetRequiredService>(); + var options = optionsMon.Get(_name); + //ILoggerFactory can be null + var factory = x.GetService(); + return new DefaultDiskCachingProvider(_name, options, factory); + }); + } + + public void WithServices(IApplicationBuilder app) + { + // Method intentionally left empty. + } + } +} diff --git a/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs b/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs new file mode 100644 index 00000000..62df72dd --- /dev/null +++ b/src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs @@ -0,0 +1,40 @@ +namespace EasyCaching.Disk +{ + using System; + using EasyCaching.Core; + using EasyCaching.Core.Configurations; + using Microsoft.Extensions.Configuration; + + public static class EasyCachingOptionsExtensions + { + /// + /// Uses the disk caching provider. + /// + /// Options. + /// Configure. + /// Name. + public static EasyCachingOptions UseDisk(this EasyCachingOptions options, Action configure, string name = EasyCachingConstValue.DefaultDiskName) + { + ArgumentCheck.NotNull(configure, nameof(configure)); + + options.RegisterExtension(new DiskOptionsExtension(name, configure)); + + return options; + } + + public static EasyCachingOptions UseDisk(this EasyCachingOptions options, IConfiguration configuration, string name = EasyCachingConstValue.DefaultDiskName, string sectionName = EasyCachingConstValue.DiskSection) + { + var dbConfig = configuration.GetSection(sectionName); + var diskOptions = new DiskOptions(); + dbConfig.Bind(diskOptions); + + void configure(DiskOptions x) + { + x.EnableLogging = diskOptions.EnableLogging; + x.MaxRdSecond = diskOptions.MaxRdSecond; + x.DBConfig = diskOptions.DBConfig; + } + return options.UseDisk(configure, name); + } + } +} diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs new file mode 100644 index 00000000..8b23d6a3 --- /dev/null +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs @@ -0,0 +1,434 @@ +namespace EasyCaching.Disk +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using EasyCaching.Core; + using MessagePack; + using Microsoft.Extensions.Logging; + + public partial class DefaultDiskCachingProvider : EasyCachingAbstractProvider + { + public override async Task BaseExistsAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = BuildMd5Path(cacheKey); + + if (!File.Exists(path)) return false; + + var val = await GetDiskCacheValueAsync(path); + + return val.Expiration > DateTimeOffset.UtcNow; + } + + public override Task BaseFlushAsync() + { + if (_options.EnableLogging) + _logger?.LogInformation("FlushAsync"); + + var md5FolderName = GetMd5Str(_name); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); + + DeleteDirectory(path); + + _cacheKeysMap.Clear(); + + return Task.CompletedTask; + } + + public override async Task>> BaseGetAllAsync(IEnumerable cacheKeys) + { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + + IDictionary> dict = new Dictionary>(); + + foreach (var item in cacheKeys) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + else + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; + } + + public override async Task> BaseGetAsync(string cacheKey, Func> dataRetriever, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var path = GetRawPath(cacheKey); + + if (File.Exists(path)) + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + return new CacheValue(t, true); + } + } + + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + // TODO: how to add mutex key here + if (!_cacheKeysMap.TryAdd($"{cacheKey}_Lock", "1")) + { + System.Threading.Thread.Sleep(_options.SleepMs); + return await GetAsync(cacheKey, dataRetriever, expiration); + } + + var res = await dataRetriever(); + + if (res != null) + { + await SetAsync(cacheKey, res, expiration); + //remove mutex key + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); + return new CacheValue(res, true); + } + else + { + //remove mutex key + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); + return CacheValue.NoValue; + } + } + + public override async Task BaseGetAsync(string cacheKey, Type type) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + return null; + } + + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + var t = MessagePackSerializer.NonGeneric.Deserialize(type, cached.Value); + return t; + } + else + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + + return null; + } + } + + public override async Task> BaseGetAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + return CacheValue.NoValue; + } + + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + return new CacheValue(t, true); + } + else + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + + return CacheValue.NoValue; + } + } + + public override async Task>> BaseGetByPrefixAsync(string prefix) + { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + + IDictionary> dict = new Dictionary>(); + + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); + + if (list == null || !list.Any()) return dict; + + foreach (var item in list) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + else + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; + } + + public override async Task BaseGetExpirationAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + return TimeSpan.Zero; + } + + var cached = await GetDiskCacheValueAsync(path); + + return cached.Expiration.Subtract(DateTimeOffset.UtcNow); + } + + public override Task BaseRefreshAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + // Obsolete + return Task.CompletedTask; + } + + public override Task BaseRemoveAllAsync(IEnumerable cacheKeys) + { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + + foreach (string key in cacheKeys) + { + if (string.IsNullOrWhiteSpace(key)) + continue; + + var path = GetRawPath(key); + + if (!File.Exists(path)) + { + continue; + } + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(key, out _); + } + } + + return Task.CompletedTask; + } + + public override Task BaseRemoveAsync(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + return Task.CompletedTask; + //return true; + } + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(cacheKey, out _); + } + + return Task.CompletedTask; + } + + public override Task BaseRemoveByPrefixAsync(string prefix) + { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); + + foreach (var item in list) + { + var path = BuildMd5Path(item); + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(item, out _); + } + } + + return Task.CompletedTask; + } + + public override async Task BaseSetAllAsync(IDictionary values, TimeSpan expiration) + { + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + ArgumentCheck.NotNullAndCountGTZero(values, nameof(values)); + + foreach (var item in values) + { + try + { + var (path, fileName) = GetFilePath(item.Key); + + var bytes = BuildDiskCacheValue(item.Value, expiration); + + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + } + + AppendKey(item.Key, fileName); + } + catch + { + + } + } + } + + public override async Task BaseSetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var (path, fileName) = GetFilePath(cacheKey); + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + //return true; + } + + AppendKey(cacheKey, fileName); + } + + public override async Task BaseTrySetAsync(string cacheKey, T cacheValue, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var (path, fileName) = GetFilePath(cacheKey); + + if (File.Exists(path)) + { + var cached = await GetDiskCacheValueAsync(path); + + if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) + { + return false; + } + } + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + await stream.WriteAsync(bytes, 0, bytes.Length); + AppendKey(cacheKey, fileName); + return true; + } + } + + private async Task GetDiskCacheValueAsync(string path) + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = await MessagePackSerializer.DeserializeAsync(stream); + + return cached; + } + } + } +} diff --git a/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs new file mode 100644 index 00000000..aca88afc --- /dev/null +++ b/src/EasyCaching.Disk/DefaultDiskCachingProvider.cs @@ -0,0 +1,654 @@ +namespace EasyCaching.Disk +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Security.Cryptography; + using System.Text; + using System.Threading; + using EasyCaching.Core; + using MessagePack; + using Microsoft.Extensions.Logging; + + public partial class DefaultDiskCachingProvider : EasyCachingAbstractProvider + { + /// + /// The options. + /// + private readonly DiskOptions _options; + + /// + /// The logger. + /// + private readonly ILogger _logger; + + /// + /// The cache stats. + /// + private readonly CacheStats _cacheStats; + + /// + /// The name. + /// + private readonly string _name; + + private readonly ConcurrentDictionary _cacheKeysMap; + + private Timer _saveKeyTimer; + + public DefaultDiskCachingProvider(string name, + DiskOptions options, + ILoggerFactory loggerFactory = null) + { + this._name = name; + this._options = options; + this._logger = loggerFactory?.CreateLogger(); + + this._cacheKeysMap = new ConcurrentDictionary(); + + this._cacheStats = new CacheStats(); + + this.ProviderName = _name; + this.ProviderType = CachingProviderType.Disk; + this.ProviderStats = _cacheStats; + this.ProviderMaxRdSecond = _options.MaxRdSecond; + this.IsDistributedProvider = false; + + Init(); + + _saveKeyTimer = new Timer(SaveKeyToFile, null, TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(2)); + } + + private void Init() + { + var md5FolderName = GetMd5Str(_name); + + var basePath = Path.Combine(_options.DBConfig.BasePath, md5FolderName); + + if (!Directory.Exists(basePath)) + { + Directory.CreateDirectory(basePath); + } + + var path = Path.Combine(basePath, $"key.dat"); + + if (!File.Exists(path)) + { + File.Create(path); + } + + InitCacheKey(); + } + + private void SaveKeyToFile(object state) + { + if (!_cacheKeysMap.IsEmpty) + { + var md5FolderName = GetMd5Str(_name); + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"key.dat"); + + var keys = _cacheKeysMap.Keys.ToArray(); + + var value = string.Join("\n", keys); + + var bytes = Encoding.UTF8.GetBytes(value); + + for (int i = 0; i < 3; i++) + { + try + { + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write)) + { + // batch , not all in one + stream.Write(bytes, 0, bytes.Length); + } + + break; + } + catch + { + + } + } + } + } + + private void InitCacheKey() + { + var path = BuildRawPath("key"); + + if (!File.Exists(path)) return; + + for (int i = 0; i < 3; i++) + { + try + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + _cacheKeysMap.TryAdd(line, GetMd5Str(line)); + } + } + } + + break; + } + catch + { + + } + } + } + + public override bool BaseExists(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = BuildMd5Path(cacheKey); + + if (!File.Exists(path)) return false; + + var val = GetDiskCacheValue(path); + + return val.Expiration > DateTimeOffset.UtcNow; + } + + public override void BaseFlush() + { + if (_options.EnableLogging) + _logger?.LogInformation("Flush"); + + var md5FolderName = GetMd5Str(_name); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName); + + DeleteDirectory(path); + + _cacheKeysMap.Clear(); + } + + public override CacheValue BaseGet(string cacheKey, Func dataRetriever, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var path = GetRawPath(cacheKey); + + if (File.Exists(path)) + { + var cached = GetDiskCacheValue(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + return new CacheValue(t, true); + } + } + + CacheStats.OnMiss(); + + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + // TODO: how to add mutex key here + if (!_cacheKeysMap.TryAdd($"{cacheKey}_Lock", "1")) + { + System.Threading.Thread.Sleep(_options.SleepMs); + return Get(cacheKey, dataRetriever, expiration); + } + + var res = dataRetriever(); + + if (res != null) + { + Set(cacheKey, res, expiration); + // remove mutex key + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); + + return new CacheValue(res, true); + } + else + { + // remove mutex key + _cacheKeysMap.TryRemove($"{cacheKey}_Lock", out _); + return CacheValue.NoValue; + } + } + + public override CacheValue BaseGet(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + + return CacheValue.NoValue; + } + + var cached = GetDiskCacheValue(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}"); + + CacheStats.OnHit(); + + var t = MessagePackSerializer.Deserialize(cached.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + return new CacheValue(t, true); + } + else + { + if (_options.EnableLogging) + _logger?.LogInformation($"Cache Missed : cachekey = {cacheKey}"); + + CacheStats.OnMiss(); + + return CacheValue.NoValue; + } + } + + public override IDictionary> BaseGetAll(IEnumerable cacheKeys) + { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + + IDictionary> dict = new Dictionary>(); + + foreach (var item in cacheKeys) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + else + { + var cached = GetDiskCacheValue(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; + } + + public override IDictionary> BaseGetByPrefix(string prefix) + { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + + IDictionary> dict = new Dictionary>(); + + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); + + if (list == null || !list.Any()) return dict; + + foreach (var item in list) + { + var path = GetRawPath(item); + + if (!File.Exists(path)) + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + else + { + var cached = GetDiskCacheValue(path); + + if (cached.Expiration > DateTimeOffset.UtcNow) + { + var t = MessagePackSerializer.Deserialize(cached.Value); + + if (!dict.ContainsKey(item)) + { + dict.Add(item, new CacheValue(t, true)); + } + } + else + { + if (!dict.ContainsKey(item)) + { + dict.Add(item, CacheValue.NoValue); + } + } + } + } + + return dict; + } + + public override int BaseGetCount(string prefix = "") + { + if (string.IsNullOrWhiteSpace(prefix)) + { + return _cacheKeysMap.Count; + } + else + { + return _cacheKeysMap.Count(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)); + } + } + + public override TimeSpan BaseGetExpiration(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + return TimeSpan.Zero; + } + + var cached = GetDiskCacheValue(path); + + return cached.Expiration.Subtract(DateTimeOffset.UtcNow); + } + + public override void BaseRefresh(string cacheKey, T cacheValue, TimeSpan expiration) + { + // Obsolete + } + + public override void BaseRemove(string cacheKey) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + + var path = GetRawPath(cacheKey); + + if (!File.Exists(path)) + { + return; + //return true; + } + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(cacheKey, out _); + } + } + + public override void BaseRemoveAll(IEnumerable cacheKeys) + { + ArgumentCheck.NotNullAndCountGTZero(cacheKeys, nameof(cacheKeys)); + + foreach (string key in cacheKeys) + { + if (string.IsNullOrWhiteSpace(key)) + continue; + + var path = GetRawPath(key); + + if (!File.Exists(path)) + { + continue; + } + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(key, out _); + } + } + } + + public override void BaseRemoveByPrefix(string prefix) + { + ArgumentCheck.NotNullOrWhiteSpace(prefix, nameof(prefix)); + + var list = _cacheKeysMap.Where(x => x.Key.StartsWith(prefix, StringComparison.Ordinal)).Select(x => x.Key).ToList(); + + foreach (var item in list) + { + var path = BuildMd5Path(item); + + if (DeleteFileWithRetry(path)) + { + _cacheKeysMap.TryRemove(item, out _); + } + } + } + + public override void BaseSet(string cacheKey, T cacheValue, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var (path, fileName) = GetFilePath(cacheKey); + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) + { + stream.Write(bytes, 0, bytes.Length); + //return true; + } + + AppendKey(cacheKey, fileName); + } + + public override void BaseSetAll(IDictionary values, TimeSpan expiration) + { + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + ArgumentCheck.NotNullAndCountGTZero(values, nameof(values)); + + foreach (var item in values) + { + try + { + var (path, fileName) = GetFilePath(item.Key); + + var bytes = BuildDiskCacheValue(item.Value, expiration); + + using (FileStream stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) + { + stream.Write(bytes, 0, bytes.Length); + } + + AppendKey(item.Key, fileName); + } + catch + { + + } + } + } + + public override bool BaseTrySet(string cacheKey, T cacheValue, TimeSpan expiration) + { + ArgumentCheck.NotNullOrWhiteSpace(cacheKey, nameof(cacheKey)); + ArgumentCheck.NotNull(cacheValue, nameof(cacheValue)); + ArgumentCheck.NotNegativeOrZero(expiration, nameof(expiration)); + + var (path, fileName) = GetFilePath(cacheKey); + + if (File.Exists(path)) + { + var cached = GetDiskCacheValue(path); + + if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero) + { + return false; + } + } + + var bytes = BuildDiskCacheValue(cacheValue, expiration); + + using (FileStream stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + stream.Write(bytes, 0, bytes.Length); + AppendKey(cacheKey, fileName); + return true; + } + } + + private (string path, string md5Name) GetFilePath(string key) + { + var md5FolderName = GetMd5Str(_name); + var md5FileName = GetMd5Str(key); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"{md5FileName}.dat"); + return (path, md5FileName); + } + + private string BuildRawPath(string key) + { + var md5FolderName = GetMd5Str(_name); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"{key}.dat"); + return path; + } + + private string GetRawPath(string cacheKey) + { + var path = string.Empty; + if (_cacheKeysMap.TryGetValue(cacheKey, out var fileName)) + { + path = BuildRawPath(fileName); + } + else + { + path = BuildMd5Path(cacheKey); + } + return path; + } + + + private string BuildMd5Path(string key) + { + var md5FolderName = GetMd5Str(_name); + var md5FileName = GetMd5Str(key); + + var path = Path.Combine(_options.DBConfig.BasePath, md5FolderName, $"{md5FileName}.dat"); + return path; + } + + private bool DeleteFileWithRetry(string path) + { + bool flag = false; + + for (int i = 0; i < 3; i++) + { + try + { + File.Delete(path); + flag = true; + break; + } + catch + { + //return false; + } + } + + return flag; + } + + private DiskCacheValue GetDiskCacheValue(string path) + { + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var cached = MessagePackSerializer.Deserialize(stream); + + return cached; + } + } + + private void AppendKey(string key, string md5Key) + { + _cacheKeysMap.TryAdd(key, md5Key); + } + + private byte[] BuildDiskCacheValue(T t, TimeSpan ts) + { + var value = MessagePackSerializer.Serialize(t, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + + var cached = new DiskCacheValue(value, DateTimeOffset.UtcNow.AddSeconds((int)ts.TotalSeconds)); + + var bytes = MessagePackSerializer.Serialize(cached); + + return bytes; + } + + private string GetMd5Str(string src) + { + using (MD5 md5 = MD5.Create()) + { + byte[] data = md5.ComputeHash(Encoding.UTF8.GetBytes(src)); + + StringBuilder sBuilder = new StringBuilder(64); + + for (int i = 0; i < data.Length; i++) + { + sBuilder.Append(data[i].ToString("x2")); + } + + return sBuilder.ToString(); + } + } + + private void DeleteDirectory(string path) + { + try + { + DirectoryInfo dir = new DirectoryInfo(path); + FileSystemInfo[] fileinfo = dir.GetFileSystemInfos(); + foreach (FileSystemInfo i in fileinfo) + { + if (i is DirectoryInfo) + { + DirectoryInfo subdir = new DirectoryInfo(i.FullName); + subdir.Delete(true); + } + else + { + File.Delete(i.FullName); + } + } + } + catch + { + + } + } + } +} diff --git a/src/EasyCaching.Disk/EasyCaching.Disk.csproj b/src/EasyCaching.Disk/EasyCaching.Disk.csproj new file mode 100644 index 00000000..9010896e --- /dev/null +++ b/src/EasyCaching.Disk/EasyCaching.Disk.csproj @@ -0,0 +1,39 @@ + + + + + netstandard2.0 + Catcher Wong + Catcher Wong + $(EasyCachingDiskPackageVersion) + + A simple disk(file) caching provider. + + In-Memory,LocalCache,Caching,Cache + https://github.com/dotnetcore/EasyCaching + https://github.com/dotnetcore/EasyCaching/blob/master/LICENSE + https://github.com/dotnetcore/EasyCaching + https://github.com/dotnetcore/EasyCaching + https://raw.githubusercontent.com/dotnetcore/EasyCaching/master/media/nuget-icon.png + + $(EasyCachingDiskPackageNotes) + + + + + true + $(NoWarn);1591 + + + + + + + + + + + + + + diff --git a/src/EasyCaching.Disk/Internal/DiskCacheValue.cs b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs new file mode 100644 index 00000000..de5e45aa --- /dev/null +++ b/src/EasyCaching.Disk/Internal/DiskCacheValue.cs @@ -0,0 +1,23 @@ +namespace EasyCaching.Disk +{ + using System; + using MessagePack; + + [MessagePackObject] + public class DiskCacheValue + { + [SerializationConstructor] + public DiskCacheValue(byte[] val, DateTimeOffset time) + { + Value = val; + Expiration = time; + } + + + [Key(0)] + public byte[] Value { get; private set; } + + [Key(1)] + public DateTimeOffset Expiration { get; private set; } + } +} diff --git a/src/EasyCaching.Disk/Internal/DiskDbOptions.cs b/src/EasyCaching.Disk/Internal/DiskDbOptions.cs new file mode 100644 index 00000000..b96dbf17 --- /dev/null +++ b/src/EasyCaching.Disk/Internal/DiskDbOptions.cs @@ -0,0 +1,7 @@ +namespace EasyCaching.Disk +{ + public class DiskDbOptions + { + public string BasePath { get; set; } + } +} diff --git a/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs b/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs index 730c78c8..624203e0 100644 --- a/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs +++ b/test/EasyCaching.UnitTests/CachingTests/BaseCachingProviderTest.cs @@ -609,7 +609,7 @@ protected virtual async Task GetAsync_Parallel_Should_Succeed() #region Refresh/RefreshAsync [Fact] - public void Refresh_Should_Succeed() + protected virtual void Refresh_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = "value"; @@ -629,7 +629,7 @@ public void Refresh_Should_Succeed() } [Fact] - public async Task Refresh_Async_Should_Succeed() + protected virtual async Task Refresh_Async_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = "value"; @@ -651,7 +651,7 @@ public async Task Refresh_Async_Should_Succeed() } [Fact] - public void Refresh_Value_Type_Object_Should_Succeed() + protected virtual void Refresh_Value_Type_Object_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = 100; @@ -671,7 +671,7 @@ public void Refresh_Value_Type_Object_Should_Succeed() } [Fact] - public async Task Refresh_Value_Type_Object_Async_Should_Succeed() + protected virtual async Task Refresh_Value_Type_Object_Async_Should_Succeed() { var cacheKey = $"{_nameSpace}{Guid.NewGuid().ToString()}"; var cacheValue = 100; diff --git a/test/EasyCaching.UnitTests/CachingTests/DiskCachingProviderTest.cs b/test/EasyCaching.UnitTests/CachingTests/DiskCachingProviderTest.cs new file mode 100644 index 00000000..952acb0d --- /dev/null +++ b/test/EasyCaching.UnitTests/CachingTests/DiskCachingProviderTest.cs @@ -0,0 +1,54 @@ +namespace EasyCaching.UnitTests +{ + using System; + using System.IO; + using System.Threading.Tasks; + using EasyCaching.Core; + using EasyCaching.Disk; + using Microsoft.Extensions.DependencyInjection; + using Xunit; + + public class DiskCachingProviderTest : BaseCachingProviderTest + { + public DiskCachingProviderTest() + { + IServiceCollection services = new ServiceCollection(); + services.AddEasyCaching(x => x.UseDisk(options => + { + options.MaxRdSecond = 0; + options.DBConfig = new DiskDbOptions + { + BasePath = Path.GetTempPath() + }; + + })); + IServiceProvider serviceProvider = services.BuildServiceProvider(); + _provider = serviceProvider.GetService(); + _defaultTs = TimeSpan.FromSeconds(30); + } + + [Fact(Skip = "")] + protected override void Refresh_Should_Succeed() + { + + } + + [Fact(Skip = "")] + protected override Task Refresh_Async_Should_Succeed() + { + return Task.CompletedTask; + } + + [Fact(Skip = "")] + protected override void Refresh_Value_Type_Object_Should_Succeed() + { + + } + + [Fact(Skip = "")] + protected override Task Refresh_Value_Type_Object_Async_Should_Succeed() + { + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj b/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj index 96bac279..8d8a1367 100644 --- a/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj +++ b/test/EasyCaching.UnitTests/EasyCaching.UnitTests.csproj @@ -32,6 +32,7 @@ +