Skip to content

Commit

Permalink
[feat]新增BuildDeleteSql,部分数据库支持分批删除。对超大表来说,分批删除可以缓解整表锁定问题。
Browse files Browse the repository at this point in the history
  • Loading branch information
nnhy committed Jun 3, 2024
1 parent 2db630c commit f2d429d
Show file tree
Hide file tree
Showing 9 changed files with 111 additions and 51 deletions.
17 changes: 16 additions & 1 deletion XCode/DataAccessLayer/Common/DbBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -995,7 +995,7 @@ public virtual IDataParameter CreateParameter(String name, Object? value, Type?
/// <summary>创建参数数组</summary>
/// <param name="ps"></param>
/// <returns></returns>
public virtual IDataParameter[] CreateParameters(IDictionary<String, Object>? ps) => ps?.Select(e => CreateParameter(e.Key, e.Value)).ToArray() ?? new IDataParameter[0];
public virtual IDataParameter[] CreateParameters(IDictionary<String, Object>? ps) => ps?.Select(e => CreateParameter(e.Key, e.Value)).ToArray() ?? [];

/// <summary>根据对象成员创建参数数组</summary>
/// <param name="model"></param>
Expand All @@ -1016,6 +1016,21 @@ public virtual IDataParameter[] CreateParameters(Object? model)
return list.ToArray();
}

/// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
/// <param name="tableName"></param>
/// <param name="where"></param>
/// <param name="batchSize"></param>
/// <returns>不支持分批删除时返回null</returns>
public virtual String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
{
if (batchSize > 0) return null;

var sql = $"Delete From {FormatName(tableName)}";
if (!where.IsNullOrEmpty()) sql += " Where " + where;

return sql;
}

/// <summary>是否支持Schema。默认true</summary>
public Boolean SupportSchema { get; set; } = true;
#endregion
Expand Down
7 changes: 7 additions & 0 deletions XCode/DataAccessLayer/Common/IDatabase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,13 @@ public interface IDatabase : IDisposable2
/// <returns></returns>
IDataParameter[] CreateParameters(Object? model);

/// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
/// <param name="tableName"></param>
/// <param name="where"></param>
/// <param name="batchSize"></param>
/// <returns>不支持分批删除时返回null</returns>
String? BuildDeleteSql(String tableName, String where, Int32 batchSize);

/// <summary>本连接数据只读</summary>
Boolean Readonly { get; set; }

Expand Down
32 changes: 10 additions & 22 deletions XCode/DataAccessLayer/DAL_Mapper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,7 @@ public Int32 Update(Object data, Object where, String? tableName = null)

return ExecuteWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.Execute(s, CommandType.Text, p), nameof(Update));
}

/// <summary>更细数据。无实体</summary>
/// <param name="data">实体对象</param>
/// <param name="table">表定义</param>
Expand Down Expand Up @@ -484,6 +485,13 @@ public Int32 Delete(String tableName, Object where)
{
if (tableName.IsNullOrEmpty()) throw new ArgumentNullException(nameof(tableName));

var (sql, dps) = GetDeleteSql(tableName, where);

return ExecuteWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.Execute(s, CommandType.Text, p), nameof(Delete));
}

private (String, IList<IDataParameter>) GetDeleteSql(String tableName, Object where)
{
var sb = Pool.StringBuilder.Get();
sb.Append("Delete From ");
sb.Append(tableName);
Expand All @@ -503,9 +511,8 @@ public Int32 Delete(String tableName, Object where)
sb.AppendFormat("{0}={1}", pi.Name, p.ParameterName);
}
}
var sql = sb.Put(true);

return ExecuteWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.Execute(s, CommandType.Text, p), nameof(Delete));
return (sb.Put(true), dps);
}

/// <summary>插入数据</summary>
Expand Down Expand Up @@ -596,26 +603,7 @@ public Task<Int32> DeleteAsync(String tableName, Object where)
{
if (tableName.IsNullOrEmpty()) throw new ArgumentNullException(nameof(tableName));

var sb = Pool.StringBuilder.Get();
sb.Append("Delete From ");
sb.Append(tableName);

// 带上参数化的Where条件
var dps = new List<IDataParameter>();
if (where != null)
{
sb.Append(" Where ");
var i = 0;
foreach (var pi in where.GetType().GetProperties(true))
{
if (i++ > 0) sb.Append("And ");

var p = Db.CreateParameter(pi.Name, pi.GetValue(where, null), pi.PropertyType);
dps.Add(p);
sb.AppendFormat("{0}={1}", pi.Name, p.ParameterName);
}
}
var sql = sb.Put(true);
var (sql, dps) = GetDeleteSql(tableName, where);

return ExecuteAsyncWrap(sql, "", dps.ToArray(), (ss, s, t, p) => ss.ExecuteAsync(s, CommandType.Text, p), nameof(DeleteAsync));
}
Expand Down
17 changes: 16 additions & 1 deletion XCode/DataAccessLayer/Database/MySql.cs
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ public override String FormatLike(IDataColumn column, String format, String valu
/// <param name="value">值</param>
/// <param name="type">类型</param>
/// <returns></returns>
public override IDataParameter CreateParameter(String name, Object value, Type? type = null)
public override IDataParameter CreateParameter(String name, Object? value, Type? type = null)
{
var dp = base.CreateParameter(name, value, type);

Expand Down Expand Up @@ -199,6 +199,21 @@ public override IDataParameter CreateParameter(String name, Object value, Type?
/// <returns></returns>
public override String StringConcat(String left, String right) => $"concat({(!String.IsNullOrEmpty(left) ? left : "\'\'")},{(!String.IsNullOrEmpty(right) ? right : "\'\'")})";

/// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
/// <param name="tableName"></param>
/// <param name="where"></param>
/// <param name="batchSize"></param>
/// <returns>不支持分批删除时返回null</returns>
public override String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
{
var sql = base.BuildDeleteSql(tableName, where, 0);

if (batchSize <= 0) return sql;

sql = $"{sql} limit {batchSize}";

return sql;
}
#endregion 数据库特性

#region 跨版本兼容
Expand Down
17 changes: 16 additions & 1 deletion XCode/DataAccessLayer/Database/Oracle.cs
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ public override String FormatValue(IDataColumn field, Object? value)
/// <param name="value">值</param>
/// <param name="type">类型</param>
/// <returns></returns>
public override IDataParameter CreateParameter(String name, Object value, Type type = null)
public override IDataParameter CreateParameter(String name, Object? value, Type? type = null)
{
//var type = field?.DataType;
if (type == null)
Expand Down Expand Up @@ -252,6 +252,21 @@ public override IDataParameter CreateParameter(String name, Object value, Type t
return dp;
}

/// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
/// <param name="tableName"></param>
/// <param name="where"></param>
/// <param name="batchSize"></param>
/// <returns>不支持分批删除时返回null</returns>
public override String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
{
if (batchSize <= 0) return base.BuildDeleteSql(tableName, where, 0);

var sql = $"Select RowId From {FormatName(tableName)} Where ";
if (!where.IsNullOrEmpty()) sql += "(" + where + ") And ";
sql += $"RowNum<={batchSize}";

return $"Delete From {FormatName(tableName)} Where RowId In({sql})";
}
#endregion 数据库特性

#region 关键字
Expand Down
15 changes: 15 additions & 0 deletions XCode/DataAccessLayer/Database/SqlServer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,21 @@ public override String FormatLike(IDataColumn column, String format, String valu
.Replace("_", "[_]");
return base.FormatLike(column, format, value);
}

/// <summary>生成批量删除SQL。部分数据库支持分批删除</summary>
/// <param name="tableName"></param>
/// <param name="where"></param>
/// <param name="batchSize"></param>
/// <returns>不支持分批删除时返回null</returns>
public override String? BuildDeleteSql(String tableName, String where, Int32 batchSize)
{
if (batchSize <= 0) return base.BuildDeleteSql(tableName, where, 0);

var sql = $"Delete Top {batchSize} From {FormatName(tableName)}";
if (!where.IsNullOrEmpty()) sql += " Where " + where;

return sql;
}
#endregion
}

Expand Down
8 changes: 2 additions & 6 deletions XCode/DataAccessLayer/InsertBuilder.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Data;
using System.Data.Common;
using System.Text;
using NewLife;
Expand All @@ -9,9 +7,7 @@

namespace XCode.DataAccessLayer;

/// <summary>
/// 插入Sql语句生成器
/// </summary>
/// <summary>插入Sql语句生成器</summary>
public class InsertBuilder
{
#region 属性
Expand Down
45 changes: 25 additions & 20 deletions XCode/Entity/IEntityPersistence.cs
Original file line number Diff line number Diff line change
Expand Up @@ -413,34 +413,39 @@ public virtual Int32 Update(IEntitySession session, String[] setNames, Object[]
/// <returns></returns>
public virtual Int32 Delete(IEntitySession session, String whereClause)
{
var sql = $"Delete From {session.FormatedTableName}";
if (!whereClause.IsNullOrEmpty()) sql += " Where " + whereClause;
//return session.Execute(sql);
var batchSize = session.Dal.Db.BatchSize;
if (batchSize <= 0) batchSize = XCodeSetting.Current.BatchSize;
if (batchSize <= 0) batchSize = 10_000;
var interval = XCodeSetting.Current.BatchInterval;

// MySql 支持分批删除
if (session.Dal.DbType == DatabaseType.MySql)
// 部分数据库支持分批删除
var rs = 0;
while (true)
{
var batchSize = session.Dal.Db.BatchSize;
if (batchSize <= 0) batchSize = XCodeSetting.Current.BatchSize;
if (batchSize <= 0) batchSize = 10_000;

var rs = 0;
while (true)
// 生成分批删除SQL,如果失败则退出,回归默认整体删除逻辑
var sql = session.Dal.Db.BuildDeleteSql(session.FormatedTableName, whereClause, batchSize);
if (sql.IsNullOrEmpty())
{
var rows = session.Dal.Execute($"{sql} limit {batchSize}", 5 * 60);
rs += rows;
rs = -1;
break;
}

if (rows < batchSize) break;
var rows = session.Dal.Execute(sql, 5 * 60);
rs += rows;

// 延迟一点,避免太快影响数据库性能
Thread.Sleep(100);
}
if (rows < batchSize) break;

return rs;
// 延迟一点,避免太快影响数据库性能
if (interval > 0) Thread.Sleep(interval);
}

// 加大超时时间
return session.Dal.Execute(sql, 5 * 60);
if (rs >= 0) return rs;

{
// 加大超时时间
var sql = session.Dal.Db.BuildDeleteSql(session.FormatedTableName, whereClause, 0);
return session.Dal.Execute(sql!, 5 * 60);
}
}

/// <summary>从数据库中删除指定属性列表和值列表所限定的实体对象。</summary>
Expand Down
4 changes: 4 additions & 0 deletions XCode/Setting.cs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ public class XCodeSetting : Config<XCodeSetting>
[Description("批大小。用于批量操作数据,抽取、删除、备份、恢复,默认5000")]
public Int32 BatchSize { get; set; } = 5_000;

/// <summary>批操作间隙。用于批量删除数据时的暂停间隙,单位毫秒,默认100</summary>
[Description("批操作间隙。用于批量删除数据时的暂停间隙,单位毫秒,默认100")]
public Int32 BatchInterval { get; set; } = 100;

/// <summary>命令超时。查询执行超时时间,默认0秒不限制</summary>
[Description("命令超时。查询执行超时时间,默认0秒不限制")]
public Int32 CommandTimeout { get; set; }
Expand Down

0 comments on commit f2d429d

Please sign in to comment.