[feat]新增BuildDeleteSql，部分数据库支持分批删除。对超大表来说，分批删除可以缓解整表锁定问题。

NewLifeX · Jun 3, 2024 · f2d429d · f2d429d
1 parent 2db630c
commit f2d429d
Show file tree

Hide file tree

Showing 9 changed files with 111 additions and 51 deletions.
diff --git a/XCode/DataAccessLayer/Common/DbBase.cs b/XCode/DataAccessLayer/Common/DbBase.cs
@@ -995,7 +995,7 @@ public virtual IDataParameter CreateParameter(String name, Object? value, Type?
     /// <summary>创建参数数组</summary>
     /// <param name="ps"></param>
     /// <returns></returns>
-    public virtual IDataParameter[] CreateParameters(IDictionary<String, Object>? ps) => ps?.Select(e => CreateParameter(e.Key, e.Value)).ToArray() ?? new IDataParameter[0];
+    public virtual IDataParameter[] CreateParameters(IDictionary<String, Object>? ps) => ps?.Select(e => CreateParameter(e.Key, e.Value)).ToArray() ?? [];
 
     /// <summary>根据对象成员创建参数数组</summary>
     /// <param name="model"></param>
@@ -1016,6 +1016,21 @@ public virtual IDataParameter[] CreateParameters(Object? model)
         return list.ToArray();
     }
 
+    /// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
+    /// <param name="tableName"></param>
+    /// <param name="where"></param>
+    /// <param name="batchSize"></param>
+    /// <returns>不支持分批删除时返回null</returns>
+    public virtual String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
+    {
+        if (batchSize > 0) return null;
+
+        var sql = $"Delete From {FormatName(tableName)}";
+        if (!where.IsNullOrEmpty()) sql += " Where " + where;
+
+        return sql;
+    }
+
     /// <summary>是否支持Schema。默认true</summary>
     public Boolean SupportSchema { get; set; } = true;
     #endregion

diff --git a/XCode/DataAccessLayer/Common/IDatabase.cs b/XCode/DataAccessLayer/Common/IDatabase.cs
@@ -235,6 +235,13 @@ public interface IDatabase : IDisposable2
     /// <returns></returns>
     IDataParameter[] CreateParameters(Object? model);
 
+    /// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
+    /// <param name="tableName"></param>
+    /// <param name="where"></param>
+    /// <param name="batchSize"></param>
+    /// <returns>不支持分批删除时返回null</returns>
+    String? BuildDeleteSql(String tableName, String where, Int32 batchSize);
+
     /// <summary>本连接数据只读</summary>
     Boolean Readonly { get; set; }
 

diff --git a/XCode/DataAccessLayer/DAL_Mapper.cs b/XCode/DataAccessLayer/DAL_Mapper.cs
@@ -375,6 +375,7 @@ public Int32 Update(Object data, Object where, String? tableName = null)
 
         return ExecuteWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.Execute(s, CommandType.Text, p), nameof(Update));
     }
+
     /// <summary>更细数据。无实体</summary>
     /// <param name="data">实体对象</param>
     /// <param name="table">表定义</param>
@@ -484,6 +485,13 @@ public Int32 Delete(String tableName, Object where)
     {
         if (tableName.IsNullOrEmpty()) throw new ArgumentNullException(nameof(tableName));
 
+        var (sql, dps) = GetDeleteSql(tableName, where);
+
+        return ExecuteWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.Execute(s, CommandType.Text, p), nameof(Delete));
+    }
+
+    private (String, IList<IDataParameter>) GetDeleteSql(String tableName, Object where)
+    {
         var sb = Pool.StringBuilder.Get();
         sb.Append("Delete From ");
         sb.Append(tableName);
@@ -503,9 +511,8 @@ public Int32 Delete(String tableName, Object where)
                 sb.AppendFormat("{0}={1}", pi.Name, p.ParameterName);
             }
         }
-        var sql = sb.Put(true);
 
-        return ExecuteWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.Execute(s, CommandType.Text, p), nameof(Delete));
+        return (sb.Put(true), dps);
     }
 
     /// <summary>插入数据</summary>
@@ -596,26 +603,7 @@ public Task<Int32> DeleteAsync(String tableName, Object where)
     {
         if (tableName.IsNullOrEmpty()) throw new ArgumentNullException(nameof(tableName));
 
-        var sb = Pool.StringBuilder.Get();
-        sb.Append("Delete From ");
-        sb.Append(tableName);
-
-        // 带上参数化的Where条件
-        var dps = new List<IDataParameter>();
-        if (where != null)
-        {
-            sb.Append(" Where ");
-            var i = 0;
-            foreach (var pi in where.GetType().GetProperties(true))
-            {
-                if (i++ > 0) sb.Append("And ");
-
-                var p = Db.CreateParameter(pi.Name, pi.GetValue(where, null), pi.PropertyType);
-                dps.Add(p);
-                sb.AppendFormat("{0}={1}", pi.Name, p.ParameterName);
-            }
-        }
-        var sql = sb.Put(true);
+        var (sql, dps) = GetDeleteSql(tableName, where);
 
         return ExecuteAsyncWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.ExecuteAsync(s, CommandType.Text, p), nameof(DeleteAsync));
     }

diff --git a/XCode/DataAccessLayer/Database/MySql.cs b/XCode/DataAccessLayer/Database/MySql.cs
@@ -164,7 +164,7 @@ public override String FormatLike(IDataColumn column, String format, String valu
     /// <param name="value">值</param>
     /// <param name="type">类型</param>
     /// <returns></returns>
-    public override IDataParameter CreateParameter(String name, Object value, Type? type = null)
+    public override IDataParameter CreateParameter(String name, Object? value, Type? type = null)
     {
         var dp = base.CreateParameter(name, value, type);
 
@@ -199,6 +199,21 @@ public override IDataParameter CreateParameter(String name, Object value, Type?
     /// <returns></returns>
     public override String StringConcat(String left, String right) => $"concat({(!String.IsNullOrEmpty(left) ? left : "\'\'")},{(!String.IsNullOrEmpty(right) ? right : "\'\'")})";
 
+    /// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
+    /// <param name="tableName"></param>
+    /// <param name="where"></param>
+    /// <param name="batchSize"></param>
+    /// <returns>不支持分批删除时返回null</returns>
+    public override String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
+    {
+        var sql = base.BuildDeleteSql(tableName, where, 0);
+
+        if (batchSize <= 0) return sql;
+
+        sql = $"{sql} limit {batchSize}";
+
+        return sql;
+    }
     #endregion 数据库特性
 
     #region 跨版本兼容

diff --git a/XCode/DataAccessLayer/Database/Oracle.cs b/XCode/DataAccessLayer/Database/Oracle.cs
@@ -216,7 +216,7 @@ public override String FormatValue(IDataColumn field, Object? value)
     /// <param name="value">值</param>
     /// <param name="type">类型</param>
     /// <returns></returns>
-    public override IDataParameter CreateParameter(String name, Object value, Type type = null)
+    public override IDataParameter CreateParameter(String name, Object? value, Type? type = null)
     {
         //var type = field?.DataType;
         if (type == null)
@@ -252,6 +252,21 @@ public override IDataParameter CreateParameter(String name, Object value, Type t
         return dp;
     }
 
+    /// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
+    /// <param name="tableName"></param>
+    /// <param name="where"></param>
+    /// <param name="batchSize"></param>
+    /// <returns>不支持分批删除时返回null</returns>
+    public override String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
+    {
+        if (batchSize <= 0) return base.BuildDeleteSql(tableName, where, 0);
+
+        var sql = $"Select RowId From {FormatName(tableName)} Where ";
+        if (!where.IsNullOrEmpty()) sql += "(" + where + ") And ";
+        sql += $"RowNum<={batchSize}";
+
+        return $"Delete From {FormatName(tableName)} Where RowId In({sql})";
+    }
     #endregion 数据库特性
 
     #region 关键字

diff --git a/XCode/DataAccessLayer/Database/SqlServer.cs b/XCode/DataAccessLayer/Database/SqlServer.cs
@@ -503,6 +503,21 @@ public override String FormatLike(IDataColumn column, String format, String valu
                  .Replace("_", "[_]");
         return base.FormatLike(column, format, value);
     }
+
+    /// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
+    /// <param name="tableName"></param>
+    /// <param name="where"></param>
+    /// <param name="batchSize"></param>
+    /// <returns>不支持分批删除时返回null</returns>
+    public override String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
+    {
+        if (batchSize <= 0) return base.BuildDeleteSql(tableName, where, 0);
+
+        var sql = $"Delete Top {batchSize} From {FormatName(tableName)}";
+        if (!where.IsNullOrEmpty()) sql += " Where " + where;
+
+        return sql;
+    }
     #endregion
 }
 

diff --git a/XCode/DataAccessLayer/InsertBuilder.cs b/XCode/DataAccessLayer/InsertBuilder.cs
@@ -1,6 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Data;
+using System.Data;
 using System.Data.Common;
 using System.Text;
 using NewLife;
@@ -9,9 +7,7 @@
 
 namespace XCode.DataAccessLayer;
 
-/// <summary>
-/// 插入Sql语句生成器
-/// </summary>
+/// <summary>插入Sql语句生成器</summary>
 public class InsertBuilder
 {
     #region 属性

diff --git a/XCode/Entity/IEntityPersistence.cs b/XCode/Entity/IEntityPersistence.cs
@@ -413,34 +413,39 @@ public virtual Int32 Update(IEntitySession session, String[] setNames, Object[]
     /// <returns></returns>
     public virtual Int32 Delete(IEntitySession session, String whereClause)
     {
-        var sql = $"Delete From {session.FormatedTableName}";
-        if (!whereClause.IsNullOrEmpty()) sql += " Where " + whereClause;
-        //return session.Execute(sql);
+        var batchSize = session.Dal.Db.BatchSize;
+        if (batchSize <= 0) batchSize = XCodeSetting.Current.BatchSize;
+        if (batchSize <= 0) batchSize = 10_000;
+        var interval = XCodeSetting.Current.BatchInterval;
 
-        // MySql 支持分批删除
-        if (session.Dal.DbType == DatabaseType.MySql)
+        // 部分数据库支持分批删除
+        var rs = 0;
+        while (true)
         {
-            var batchSize = session.Dal.Db.BatchSize;
-            if (batchSize <= 0) batchSize = XCodeSetting.Current.BatchSize;
-            if (batchSize <= 0) batchSize = 10_000;
-
-            var rs = 0;
-            while (true)
+            // 生成分批删除SQL，如果失败则退出，回归默认整体删除逻辑
+            var sql = session.Dal.Db.BuildDeleteSql(session.FormatedTableName, whereClause, batchSize);
+            if (sql.IsNullOrEmpty())
             {
-                var rows = session.Dal.Execute($"{sql} limit {batchSize}", 5 * 60);
-                rs += rows;
+                rs = -1;
+                break;
+            }
 
-                if (rows < batchSize) break;
+            var rows = session.Dal.Execute(sql, 5 * 60);
+            rs += rows;
 
-                // 延迟一点，避免太快影响数据库性能
-                Thread.Sleep(100);
-            }
+            if (rows < batchSize) break;
 
-            return rs;
+            // 延迟一点，避免太快影响数据库性能
+            if (interval > 0) Thread.Sleep(interval);
         }
 
-        // 加大超时时间
-        return session.Dal.Execute(sql, 5 * 60);
+        if (rs >= 0) return rs;
+
+        {
+            // 加大超时时间
+            var sql = session.Dal.Db.BuildDeleteSql(session.FormatedTableName, whereClause, 0);
+            return session.Dal.Execute(sql!, 5 * 60);
+        }
     }
 
     /// <summary>从数据库中删除指定属性列表和值列表所限定的实体对象。</summary>

diff --git a/XCode/Setting.cs b/XCode/Setting.cs
@@ -57,6 +57,10 @@ public class XCodeSetting : Config<XCodeSetting>
     [Description("批大小。用于批量操作数据，抽取、删除、备份、恢复，默认5000")]
     public Int32 BatchSize { get; set; } = 5_000;
 
+    /// <summary>批操作间隙。用于批量删除数据时的暂停间隙，单位毫秒，默认100</summary>
+    [Description("批操作间隙。用于批量删除数据时的暂停间隙，单位毫秒，默认100")]
+    public Int32 BatchInterval { get; set; } = 100;
+
     /// <summary>命令超时。查询执行超时时间，默认0秒不限制</summary>
     [Description("命令超时。查询执行超时时间，默认0秒不限制")]
     public Int32 CommandTimeout { get; set; }