Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# wtm custom
.worktrees/

demo/WalkingTec.Mvvm.Demo/wwwroot/layuiadminsrc
.Publish/
Expand Down
178 changes: 102 additions & 76 deletions src/WalkingTec.Mvvm.Core/Analysis/AnalysisQueryEngine.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;

Expand All @@ -17,6 +17,21 @@ public class AnalysisQueryEngine
// 防止全表載入造成記憶體耗盡(C-1)
private const int MaxMaterializeRows = 50_000;

private readonly IAnalysisCache? _cache;

/// <summary>
/// 建立不帶快取的引擎(向後相容)。
/// </summary>
public AnalysisQueryEngine() : this(null) { }

/// <summary>
/// 建立帶快取的引擎。傳入 null 表示不使用快取。
/// </summary>
public AnalysisQueryEngine(IAnalysisCache? cache)
{
_cache = cache;
}

/// <summary>
/// 執行分析查詢,回傳聚合結果。
/// </summary>
Expand All @@ -28,6 +43,13 @@ public AnalysisQueryResponse Execute<TModel>(
var wl = whitelist.ToDictionary(f => f.FieldName);
ValidateFields(req, wl);

// 先計算 hash,用於快取查詢(hash 僅由 request 決定,與資料無關)
var queryHash = ComputeHash(req);

// 快取命中時直接回傳
if (_cache != null && _cache.TryGet(queryHash, out var cached) && cached != null)
return cached;

var filtered = ApplyFilters(baseQuery, req.Filters, wl);
var rows = ExecuteGroupBy(filtered, req, wl);
int totalCount = rows.Count; // 截斷前的真實筆數(I-3)
Expand All @@ -38,41 +60,45 @@ public AnalysisQueryResponse Execute<TModel>(
.Concat(req.Measures.Select(m => $"{m.Field}_{m.Func}"))
.ToList();

return new AnalysisQueryResponse
var response = new AnalysisQueryResponse
{
Columns = columns,
Rows = rows,
TotalCount = totalCount,
Truncated = truncated,
QueryHash = ComputeHash(req)
QueryHash = queryHash
};

_cache?.Set(queryHash, response);

return response;
}

/// <summary>
/// 非泛型入口,供 Controller 使用(IQueryable 無型別參數時)。
/// </summary>
public AnalysisQueryResponse ExecuteDynamic(
IQueryable baseQuery,
AnalysisQueryRequest req,
IEnumerable<AnalysisFieldMeta> whitelist)
{
var elementType = baseQuery.ElementType;
var method = typeof(AnalysisQueryEngine)
.GetMethod(nameof(Execute));
if (method is null)
throw new InvalidOperationException("Execute method not found.");
method = method.MakeGenericMethod(elementType);
try
{
var result = method.Invoke(this, new object[] { baseQuery, req, whitelist }) as AnalysisQueryResponse;
if (result is null)
throw new InvalidOperationException("ExecuteDynamic did not return a valid AnalysisQueryResponse.");
return result;
}
catch (System.Reflection.TargetInvocationException ex)
{
throw ex.InnerException ?? ex;
}
public AnalysisQueryResponse ExecuteDynamic(
IQueryable baseQuery,
AnalysisQueryRequest req,
IEnumerable<AnalysisFieldMeta> whitelist)
{
var elementType = baseQuery.ElementType;
var method = typeof(AnalysisQueryEngine)
.GetMethod(nameof(Execute));
if (method is null)
throw new InvalidOperationException("Execute method not found.");
method = method.MakeGenericMethod(elementType);
try
{
var result = method.Invoke(this, new object[] { baseQuery, req, whitelist }) as AnalysisQueryResponse;
if (result is null)
throw new InvalidOperationException("ExecuteDynamic did not return a valid AnalysisQueryResponse.");
return result;
}
catch (System.Reflection.TargetInvocationException ex)
{
throw ex.InnerException ?? ex;
}
}

private static void ValidateFields(AnalysisQueryRequest req, Dictionary<string, AnalysisFieldMeta> wl)
Expand Down Expand Up @@ -130,17 +156,17 @@ private static IQueryable<TModel> ApplyFilters<TModel>(
case FilterOperator.Lte:
body = Expression.LessThanOrEqual(prop, constant);
break;
case FilterOperator.Contains:
if (meta.ClrType != typeof(string))
throw new InvalidOperationException(
$"Contains 只適用於字串欄位,'{filter.Field}' 的型別為 {meta.ClrType.Name}。");
var containsMethod = typeof(string).GetMethod(nameof(string.Contains), new[] { typeof(string) });
if (containsMethod is null)
throw new InvalidOperationException("string.Contains(string) method not found.");
body = Expression.Call(prop,
containsMethod,
constant);
break;
case FilterOperator.Contains:
if (meta.ClrType != typeof(string))
throw new InvalidOperationException(
$"Contains 只適用於字串欄位,'{filter.Field}' 的型別為 {meta.ClrType.Name}。");
var containsMethod = typeof(string).GetMethod(nameof(string.Contains), new[] { typeof(string) });
if (containsMethod is null)
throw new InvalidOperationException("string.Contains(string) method not found.");
body = Expression.Call(prop,
containsMethod,
constant);
break;
default:
throw new InvalidOperationException($"Operator '{filter.Operator}' is not supported.");
}
Expand All @@ -150,11 +176,11 @@ private static IQueryable<TModel> ApplyFilters<TModel>(
return query;
}

private static object? ConvertValue(string value, Type targetType)
{
var underlying = Nullable.GetUnderlyingType(targetType) ?? targetType;
try
{
private static object? ConvertValue(string value, Type targetType)
{
var underlying = Nullable.GetUnderlyingType(targetType) ?? targetType;
try
{
return Convert.ChangeType(value, underlying);
}
catch (Exception ex) when (ex is InvalidCastException or FormatException or OverflowException)
Expand All @@ -163,33 +189,33 @@ private static IQueryable<TModel> ApplyFilters<TModel>(
}
}

private static List<Dictionary<string, object?>> ExecuteGroupBy<TModel>(
IQueryable<TModel> query,
AnalysisQueryRequest req,
Dictionary<string, AnalysisFieldMeta> wl)
private static List<Dictionary<string, object?>> ExecuteGroupBy<TModel>(
IQueryable<TModel> query,
AnalysisQueryRequest req,
Dictionary<string, AnalysisFieldMeta> wl)
{
// Phase 1: materialise then group in-process (SQLite + InMemory safe)
// 限制載入筆數防止 OOM(C-1);超出上限時查詢結果可能不完整,由呼叫端決策
var items = query.Take(MaxMaterializeRows).ToList();

return items
.GroupBy(row => BuildGroupKey(row, req.Dimensions))
.Take(MaxRows + 1)
.Select(g =>
{
var dict = new Dictionary<string, object?>();
var keyParts = g.Key.Split('\0');
for (int i = 0; i < req.Dimensions.Count; i++)
dict[req.Dimensions[i]] = keyParts[i];

foreach (var m in req.Measures)
{
var propInfo = typeof(TModel).GetProperty(m.Field);
if (propInfo is null)
throw new InvalidOperationException($"Property '{m.Field}' not found on {typeof(TModel).Name}.");
// 過濾 null 值,避免 nullable 型別的 Convert.ToDecimal 例外(I-10)
var values = g
.Select(row => propInfo.GetValue(row))
.Take(MaxRows + 1)
.Select(g =>
{
var dict = new Dictionary<string, object?>();
var keyParts = g.Key.Split('\0');
for (int i = 0; i < req.Dimensions.Count; i++)
dict[req.Dimensions[i]] = keyParts[i];
foreach (var m in req.Measures)
{
var propInfo = typeof(TModel).GetProperty(m.Field);
if (propInfo is null)
throw new InvalidOperationException($"Property '{m.Field}' not found on {typeof(TModel).Name}.");
// 過濾 null 值,避免 nullable 型別的 Convert.ToDecimal 例外(I-10)
var values = g
.Select(row => propInfo.GetValue(row))
.Where(v => v != null)
.Select(v => Convert.ToDecimal(v))
.ToList();
Expand All @@ -206,18 +232,18 @@ private static IQueryable<TModel> ApplyFilters<TModel>(
dict[$"{m.Field}_{m.Func}"] = aggValue;
}
return dict;
})
.ToList();
}

private static string BuildGroupKey<TModel>(TModel row, List<string> dimensions)
=> string.Join('\0', dimensions.Select(d =>
{
var propInfo = typeof(TModel).GetProperty(d);
if (propInfo is null)
throw new InvalidOperationException($"Property '{d}' not found on {typeof(TModel).Name}.");
return propInfo.GetValue(row)?.ToString() ?? string.Empty;
}));
})
.ToList();
}
private static string BuildGroupKey<TModel>(TModel row, List<string> dimensions)
=> string.Join('\0', dimensions.Select(d =>
{
var propInfo = typeof(TModel).GetProperty(d);
if (propInfo is null)
throw new InvalidOperationException($"Property '{d}' not found on {typeof(TModel).Name}.");
return propInfo.GetValue(row)?.ToString() ?? string.Empty;
}));

private static string ComputeHash(AnalysisQueryRequest req)
{
Expand Down
16 changes: 16 additions & 0 deletions src/WalkingTec.Mvvm.Core/Analysis/IAnalysisCache.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#nullable enable
using System;

namespace WalkingTec.Mvvm.Core.Analysis
{
/// <summary>
/// 分析查詢快取介面,用於快取 QueryHash 對應的查詢結果。
/// </summary>
public interface IAnalysisCache
{
bool TryGet(string queryHash, out AnalysisQueryResponse? cached);
void Set(string queryHash, AnalysisQueryResponse response, TimeSpan? ttl = null);
void Invalidate(string queryHash);
void InvalidateAll();
}
}
62 changes: 62 additions & 0 deletions src/WalkingTec.Mvvm.Core/Analysis/MemoryAnalysisCache.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#nullable enable
using System;
using System.Threading;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Primitives;

namespace WalkingTec.Mvvm.Core.Analysis
{
/// <summary>
/// 基於 IMemoryCache 的分析查詢快取實作。
/// 使用 CancellationTokenSource 實現 InvalidateAll(IMemoryCache 無 Clear 方法)。
/// </summary>
public class MemoryAnalysisCache : IAnalysisCache
{
private readonly IMemoryCache _cache;
private readonly TimeSpan _defaultTtl;
private CancellationTokenSource _cts;
private readonly object _lock = new();

public MemoryAnalysisCache(IMemoryCache cache, TimeSpan? defaultTtl = null)
{
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_defaultTtl = defaultTtl ?? TimeSpan.FromMinutes(5);
_cts = new CancellationTokenSource();
}

public bool TryGet(string queryHash, out AnalysisQueryResponse? cached)
{
return _cache.TryGetValue(queryHash, out cached);
}

public void Set(string queryHash, AnalysisQueryResponse response, TimeSpan? ttl = null)
{
var options = new MemoryCacheEntryOptions
{
AbsoluteExpirationRelativeToNow = ttl ?? _defaultTtl
};

lock (_lock)
{
options.AddExpirationToken(new CancellationChangeToken(_cts.Token));
}

_cache.Set(queryHash, response, options);
}

public void Invalidate(string queryHash)
{
_cache.Remove(queryHash);
}

public void InvalidateAll()
{
lock (_lock)
{
_cts.Cancel();
_cts.Dispose();
_cts = new CancellationTokenSource();
}
}
}
}
21 changes: 21 additions & 0 deletions src/WalkingTec.Mvvm.Core/Analysis/NullAnalysisCache.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#nullable enable
using System;

namespace WalkingTec.Mvvm.Core.Analysis
{
/// <summary>
/// 不執行任何快取的空實作,用於停用快取時。
/// </summary>
public class NullAnalysisCache : IAnalysisCache
{
public bool TryGet(string queryHash, out AnalysisQueryResponse? cached)
{
cached = null;
return false;
}

public void Set(string queryHash, AnalysisQueryResponse response, TimeSpan? ttl = null) { }
public void Invalidate(string queryHash) { }
public void InvalidateAll() { }
}
}
1 change: 1 addition & 0 deletions src/WalkingTec.Mvvm.Core/WalkingTec.Mvvm.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
<ItemGroup>
<PackageReference Include="Aliyun.OSS.SDK.NetCore" Version="2.13.0" />
<PackageReference Include="Microsoft.AspNetCore.Http" Version="2.1.34" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Identity.Core" Version="8.0.22" />
<PackageReference Include="NPOI" Version="2.7.6" />
<PackageReference Include="Fare" Version="2.2.1" />
Expand Down
2 changes: 2 additions & 0 deletions src/WalkingTec.Mvvm.Mvc/Helper/FrameworkServiceExtension.cs
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,8 @@ public static IServiceCollection AddWtmContext(this IServiceCollection services,
var analysisRegistry = new WalkingTec.Mvvm.Core.Analysis.AnalysisVmRegistry();
analysisRegistry.Build(AppDomain.CurrentDomain.GetAssemblies());
services.AddSingleton(analysisRegistry);
services.AddMemoryCache();
services.AddSingleton<WalkingTec.Mvvm.Core.Analysis.IAnalysisCache, WalkingTec.Mvvm.Core.Analysis.MemoryAnalysisCache>();
var cs = conf.Connections.Where(x => x.Enabled).ToList();
foreach (var item in cs)
{
Expand Down
Loading
Loading