From 5276d9b713ada4912d6d4c419d987c0499a22343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=99=BA=E8=83=BD=E5=A4=A7=E7=9F=B3=E5=A4=B4?= Date: Tue, 27 Feb 2024 21:01:59 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E5=99=A8=E4=BB=8EStartTime=E5=BC=80=E5=A7=8B=E5=8F=96=EF=BC=8C?= =?UTF-8?q?=E6=98=AF=E5=A4=A7=E4=BA=8E=E7=AD=89=E4=BA=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- XCode/DataAccessLayer/DbPackage.cs | 14 +- .../\345\234\260\345\214\272.Biz.cs" | 2 +- XCode/Transform/ExtractSetting.cs | 180 +++++++++-------- XCode/Transform/IETLModule.cs | 71 +++---- XCode/Transform/IETLStat.cs | 93 ++++----- XCode/Transform/IExtracter.cs | 117 ++++++----- XCode/Transform/IdExtracter.cs | 131 ++++++------- XCode/Transform/PagingExtracter.cs | 125 ++++++------ XCode/Transform/TimeExtracter.cs | 185 +++++++++--------- 9 files changed, 450 insertions(+), 468 deletions(-) diff --git a/XCode/DataAccessLayer/DbPackage.cs b/XCode/DataAccessLayer/DbPackage.cs index 6cc8b236f..347ae180c 100644 --- a/XCode/DataAccessLayer/DbPackage.cs +++ b/XCode/DataAccessLayer/DbPackage.cs @@ -168,9 +168,19 @@ protected virtual IExtracter GetExtracter(IDataTable table) return new IdExtracter(Dal, tableName, id); // 时间索引抽取 - var time = table.Indexes.FirstOrDefault(e => table.GetColumn(e.Columns[0]).DataType == typeof(DateTime)); + IDataColumn? time = null; + foreach (var dx in table.Indexes) + { + var column = table.GetColumn(dx.Columns[0]); + if (column != null && column.DataType == typeof(DateTime)) + { + time = column; + break; + } + } + //var time = table.Indexes.FirstOrDefault(e => table.GetColumn(e.Columns[0])?.DataType == typeof(DateTime)); if (time != null) - return new TimeExtracter(Dal, tableName, table.GetColumn(time.Columns[0])); + return new TimeExtracter(Dal, tableName, time); // 主键分页功能 var pk = table.Columns.FirstOrDefault(e => e.PrimaryKey); diff --git "a/XCode/Membership/\345\234\260\345\214\272.Biz.cs" "b/XCode/Membership/\345\234\260\345\214\272.Biz.cs" index 556a9d343..574c4ca67 100644 --- "a/XCode/Membership/\345\234\260\345\214\272.Biz.cs" +++ "b/XCode/Membership/\345\234\260\345\214\272.Biz.cs" @@ -892,7 +892,7 @@ public static Int32 FetchAndSave(String? url = null) { //if (url.IsNullOrEmpty()) url = "http://www.mca.gov.cn/article/sj/xzqh/2020/2020/2020092500801.html"; //if (url.IsNullOrEmpty()) url = "https://www.mca.gov.cn/mzsj/xzqh/2022/202201xzqh.html"; - if (url.IsNullOrEmpty()) url = "https://x.newlifex.com/202201xzqh.htm"; + if (url.IsNullOrEmpty()) url = "http://x.newlifex.com/202201xzqh.htm"; var http = new HttpClient(); var html = Task.Run(() => http.GetStringAsync(url)).Result; diff --git a/XCode/Transform/ExtractSetting.cs b/XCode/Transform/ExtractSetting.cs index 753bd32d0..922c28efd 100644 --- a/XCode/Transform/ExtractSetting.cs +++ b/XCode/Transform/ExtractSetting.cs @@ -1,108 +1,106 @@ -using System; -using System.Runtime.Serialization; +using System.Runtime.Serialization; using System.Xml.Serialization; -namespace XCode.Transform +namespace XCode.Transform; + +/// 数据抽取参数 +public interface IExtractSetting { - /// 数据抽取参数 - public interface IExtractSetting - { - /// 开始。大于等于 - DateTime Start { get; set; } + /// 开始。大于等于 + DateTime Start { get; set; } + + /// 结束。小于 + DateTime End { get; set; } - /// 结束。小于 - DateTime End { get; set; } + /// 时间偏移。距离实时时间的秒数，部分业务不能跑到实时 + Int32 Offset { get; set; } - /// 时间偏移。距离实时时间的秒数，部分业务不能跑到实时 - Int32 Offset { get; set; } + /// 开始行。分页 + Int32 Row { get; set; } - /// 开始行。分页 - Int32 Row { get; set; } + /// 步进。最大区间大小，秒 + Int32 Step { get; set; } - /// 步进。最大区间大小，秒 - Int32 Step { get; set; } + /// 批大小 + Int32 BatchSize { get; set; } - /// 批大小 - Int32 BatchSize { get; set; } + ///// 启用 + //Boolean Enable { get; set; } +} - ///// 启用 - //Boolean Enable { get; set; } +/// 数据抽取参数 +public class ExtractSetting : IExtractSetting +{ + #region 属性 + /// 开始。大于等于 + [XmlIgnore, IgnoreDataMember] + public DateTime Start { get; set; } + + /// 结束。小于 + [XmlIgnore, IgnoreDataMember] + public DateTime End { get; set; } + + /// 时间偏移。距离实时时间的秒数，部分业务不能跑到实时 + [XmlIgnore, IgnoreDataMember] + public Int32 Offset { get; set; } + + /// 开始行。分页 + [XmlIgnore, IgnoreDataMember] + public Int32 Row { get; set; } + + /// 步进。最大区间大小，秒 + [XmlIgnore, IgnoreDataMember] + public Int32 Step { get; set; } + + /// 批大小 + [XmlIgnore, IgnoreDataMember] + public Int32 BatchSize { get; set; } = 5000; + + ///// 启用 + //public Boolean Enable { get; set; } = true; + #endregion + + #region 构造 + /// 实例化 + public ExtractSetting() { } + + /// 实例化 + /// + public ExtractSetting(IExtractSetting set) + { + this.Copy(set); } + #endregion +} - /// 数据抽取参数 - public class ExtractSetting : IExtractSetting +/// 抽取参数帮助类 +public static class ExtractSettingHelper +{ + /// 拷贝设置参数 + /// + /// + public static IExtractSetting Copy(this IExtractSetting src, IExtractSetting set) { - #region 属性 - /// 开始。大于等于 - [XmlIgnore, IgnoreDataMember] - public DateTime Start { get; set; } - - /// 结束。小于 - [XmlIgnore, IgnoreDataMember] - public DateTime End { get; set; } - - /// 时间偏移。距离实时时间的秒数，部分业务不能跑到实时 - [XmlIgnore, IgnoreDataMember] - public Int32 Offset { get; set; } - - /// 开始行。分页 - [XmlIgnore, IgnoreDataMember] - public Int32 Row { get; set; } - - /// 步进。最大区间大小，秒 - [XmlIgnore, IgnoreDataMember] - public Int32 Step { get; set; } - - /// 批大小 - [XmlIgnore, IgnoreDataMember] - public Int32 BatchSize { get; set; } = 5000; - - ///// 启用 - //public Boolean Enable { get; set; } = true; - #endregion - - #region 构造 - /// 实例化 - public ExtractSetting() { } - - /// 实例化 - /// - public ExtractSetting(IExtractSetting set) - { - this.Copy(set); - } - #endregion + if (src == null || set == null) return src; + + src.Start = set.Start; + src.End = set.End; + src.Row = set.Row; + src.Step = set.Step; + src.BatchSize = set.BatchSize; + //src.Enable = set.Enable; + + return src; } - /// 抽取参数帮助类 - public static class ExtractSettingHelper + /// 克隆一份设置参数 + /// + /// + public static IExtractSetting Clone(this IExtractSetting src) { - /// 拷贝设置参数 - /// - /// - public static IExtractSetting Copy(this IExtractSetting src, IExtractSetting set) - { - if (src == null | set == null) return src; - - src.Start = set.Start; - src.End = set.End; - src.Row = set.Row; - src.Step = set.Step; - src.BatchSize = set.BatchSize; - //src.Enable = set.Enable; - - return src; - } - - /// 克隆一份设置参数 - /// - /// - public static IExtractSetting Clone(this IExtractSetting src) - { - var set = new ExtractSetting(); - set.Copy(src); - - return set; - } + var set = new ExtractSetting(); + set.Copy(src); + + return set; } } \ No newline at end of file diff --git a/XCode/Transform/IETLModule.cs b/XCode/Transform/IETLModule.cs index 054f9afde..468110d98 100644 --- a/XCode/Transform/IETLModule.cs +++ b/XCode/Transform/IETLModule.cs @@ -1,42 +1,35 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +namespace XCode.Transform; -namespace XCode.Transform +/// 数据抽取模块，用于自定义抽取过程中各个环节 +public interface IETLModule { - /// 数据抽取模块，用于自定义抽取过程中各个环节 - public interface IETLModule - { - /// 开始调度 - void Start(); - - /// 停止调度 - void Stop(); - - /// 首次初始化任务 - void Init(); - - /// 单批数据处理前 - /// 数据上下文 - /// - Boolean Processing(DataContext ctx); - - /// 单批数据处理后 - /// 数据上下文 - void Processed(DataContext ctx); - - /// 抽取完成 - /// 数据上下文 - void Fetched(DataContext ctx); - - /// 实体列表完成后 - /// 数据上下文 - void OnFinished(DataContext ctx); - - /// 出错 - /// 数据上下文 - void OnError(DataContext ctx); - } + /// 开始调度 + void Start(); + + /// 停止调度 + void Stop(); + + /// 首次初始化任务 + void Init(); + + /// 单批数据处理前 + /// 数据上下文 + /// + Boolean Processing(DataContext ctx); + + /// 单批数据处理后 + /// 数据上下文 + void Processed(DataContext ctx); + + /// 抽取完成 + /// 数据上下文 + void Fetched(DataContext ctx); + + /// 实体列表完成后 + /// 数据上下文 + void OnFinished(DataContext ctx); + + /// 出错 + /// 数据上下文 + void OnError(DataContext ctx); } \ No newline at end of file diff --git a/XCode/Transform/IETLStat.cs b/XCode/Transform/IETLStat.cs index 40834d911..b2d393754 100644 --- a/XCode/Transform/IETLStat.cs +++ b/XCode/Transform/IETLStat.cs @@ -1,68 +1,61 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +namespace XCode.Transform; -namespace XCode.Transform +/// ETL统计接口 +public interface IETLStat { - /// ETL统计接口 - public interface IETLStat - { - #region 性能指标 - /// 总数 - Int32 Total { get; set; } + #region 性能指标 + /// 总数 + Int32 Total { get; set; } - /// 成功 - Int32 Success { get; set; } + /// 成功 + Int32 Success { get; set; } - /// 改变数 - Int32 Changes { get; set; } + /// 改变数 + Int32 Changes { get; set; } - /// 次数 - Int32 Times { get; set; } + /// 次数 + Int32 Times { get; set; } - ///// 速度 - //Int32 Speed { get; set; } + ///// 速度 + //Int32 Speed { get; set; } - ///// 抽取速度 - //Int32 FetchSpeed { get; set; } + ///// 抽取速度 + //Int32 FetchSpeed { get; set; } - /// 错误 - Int32 Error { get; set; } + /// 错误 + Int32 Error { get; set; } - /// 错误内容 - String Message { get; set; } - #endregion - } + /// 错误内容 + String Message { get; set; } + #endregion +} - /// ETL统计 - public class ETLStat : IETLStat - { - #region 性能指标 - /// 总数 - public Int32 Total { get; set; } +/// ETL统计 +public class ETLStat : IETLStat +{ + #region 性能指标 + /// 总数 + public Int32 Total { get; set; } - /// 成功 - public Int32 Success { get; set; } + /// 成功 + public Int32 Success { get; set; } - /// 改变数 - public Int32 Changes { get; set; } + /// 改变数 + public Int32 Changes { get; set; } - /// 次数 - public Int32 Times { get; set; } + /// 次数 + public Int32 Times { get; set; } - ///// 速度 - //public Int32 Speed { get; set; } + ///// 速度 + //public Int32 Speed { get; set; } - ///// 抽取速度 - //public Int32 FetchSpeed { get; set; } + ///// 抽取速度 + //public Int32 FetchSpeed { get; set; } - /// 错误 - public Int32 Error { get; set; } + /// 错误 + public Int32 Error { get; set; } - /// 错误内容 - public String Message { get; set; } - #endregion - } + /// 错误内容 + public String Message { get; set; } + #endregion } \ No newline at end of file diff --git a/XCode/Transform/IExtracter.cs b/XCode/Transform/IExtracter.cs index 653e5ac8d..a2c926e25 100644 --- a/XCode/Transform/IExtracter.cs +++ b/XCode/Transform/IExtracter.cs @@ -1,78 +1,75 @@ -using System; -using System.Collections.Generic; -using NewLife; +using NewLife; using NewLife.Log; using XCode.Configuration; -namespace XCode.Transform +namespace XCode.Transform; + +/// 数据抽取接口 +/// +public interface IExtracter { - /// 数据抽取接口 - /// - public interface IExtracter - { - #region 属性 - /// 开始行。分页时表示偏移行数，自增时表示下一个编号，默认0 - Int64 Row { get; set; } - #endregion + #region 属性 + /// 开始行。分页时表示偏移行数，自增时表示下一个编号，默认0 + Int64 Row { get; set; } + #endregion - #region 抽取数据 - /// 迭代抽取数据 - /// - IEnumerable Fetch(); - #endregion - } + #region 抽取数据 + /// 迭代抽取数据 + /// + IEnumerable Fetch(); + #endregion +} - /// 抽取器基类 - public abstract class ExtracterBase - { - #region 属性 - /// 名称 - public String Name { get; set; } +/// 抽取器基类 +public abstract class ExtracterBase +{ + #region 属性 + /// 名称 + public String Name { get; set; } - /// 实体工厂 - public IEntityFactory Factory { get; set; } + /// 实体工厂 + public IEntityFactory Factory { get; set; } - /// 获取 或 设置 时间字段 - public String FieldName { get; set; } + /// 获取 或 设置 时间字段 + public String FieldName { get; set; } - /// 附加条件 - public String Where { get; set; } + /// 附加条件 + public String Where { get; set; } - /// 时间字段 - public FieldItem Field { get; set; } + /// 时间字段 + public FieldItem Field { get; set; } - /// 排序 - public String OrderBy { get; set; } + /// 排序 + public String OrderBy { get; set; } - /// 选择列 - public String Selects { get; set; } - #endregion + /// 选择列 + public String Selects { get; set; } + #endregion - #region 构造 - /// 实例化时基抽取算法 - public ExtracterBase() => Name = GetType().Name.TrimEnd("Extracter"); - #endregion + #region 构造 + /// 实例化时基抽取算法 + public ExtracterBase() => Name = GetType().Name.TrimEnd("Extracter"); + #endregion - #region 方法 - /// 初始化 - public virtual void Init() - { - var fi = Field; - // 自动找字段 - if (fi == null && !FieldName.IsNullOrEmpty()) fi = Field = Factory?.Table.FindByName(FieldName); + #region 方法 + /// 初始化 + public virtual void Init() + { + var fi = Field; + // 自动找字段 + if (fi == null && !FieldName.IsNullOrEmpty()) fi = Field = Factory?.Table.FindByName(FieldName); - if (fi == null) throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的排序字段！"); - } - #endregion + if (fi == null) throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的排序字段！"); + } + #endregion - #region 日志 - /// 日志 - public ILog Log { get; set; } = Logger.Null; + #region 日志 + /// 日志 + public ILog Log { get; set; } = Logger.Null; - /// 写日志 - /// - /// - public void WriteLog(String format, params Object?[] args) => Log?.Info(format, args); - #endregion - } + /// 写日志 + /// + /// + public void WriteLog(String format, params Object?[] args) => Log?.Info(format, args); + #endregion } \ No newline at end of file diff --git a/XCode/Transform/IdExtracter.cs b/XCode/Transform/IdExtracter.cs index c69e66d40..a316a61ec 100644 --- a/XCode/Transform/IdExtracter.cs +++ b/XCode/Transform/IdExtracter.cs @@ -1,88 +1,85 @@ -using System; -using System.Collections.Generic; -using NewLife; +using NewLife; using NewLife.Data; using XCode.DataAccessLayer; -namespace XCode.Transform +namespace XCode.Transform; + +/// 整型主键数据抽取器 +/// +/// 适用于带有自增字段或雪花Id字段的数据抽取器，速度飞快。 +/// +public class IdExtracter : IExtracter { - /// 整型主键数据抽取器 - /// - /// 适用于带有自增字段或雪花Id字段的数据抽取器，速度飞快。 - /// - public class IdExtracter : IExtracter - { - #region 属性 - /// 数据层 - public DAL Dal { get; set; } + #region 属性 + /// 数据层 + public DAL Dal { get; set; } - /// 查询表达式 - public SelectBuilder Builder { get; set; } + /// 查询表达式 + public SelectBuilder Builder { get; set; } - /// Id字段 - public IDataColumn Field { get; set; } + /// Id字段 + public IDataColumn Field { get; set; } - /// 开始行。默认0 - public Int64 Row { get; set; } + /// 开始行。默认0 + public Int64 Row { get; set; } - /// 批大小。默认5000 - public Int32 BatchSize { get; set; } = 5000; + /// 批大小。默认5000 + public Int32 BatchSize { get; set; } = 5000; - /// 总行数 - public Int32 TotalCount { get; set; } - #endregion + /// 总行数 + public Int32 TotalCount { get; set; } + #endregion - #region 构造 - /// 实例化自增抽取器 - public IdExtracter() { } + #region 构造 + /// 实例化自增抽取器 + public IdExtracter() { } - /// 实例化自增抽取器 - /// - /// - /// - public IdExtracter(DAL dal, String tableName, IDataColumn field) - { - Dal = dal; - Builder = new SelectBuilder { Table = tableName, OrderBy = field.ColumnName + " asc" }; - Field = field; - BatchSize = dal.Db.BatchSize; - if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; - } - #endregion + /// 实例化自增抽取器 + /// + /// + /// + public IdExtracter(DAL dal, String tableName, IDataColumn field) + { + Dal = dal; + Builder = new SelectBuilder { Table = tableName, OrderBy = field.ColumnName + " asc" }; + Field = field; + BatchSize = dal.Db.BatchSize; + if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; + } + #endregion - #region 抽取数据 - /// 迭代抽取数据 - /// - public virtual IEnumerable Fetch() + #region 抽取数据 + /// 迭代抽取数据 + /// + public virtual IEnumerable Fetch() + { + var field = Field; + var db = Dal.Db; + var name = db.FormatName(field); + while (true) { - var field = Field; - var db = Dal.Db; - var name = db.FormatName(field); - while (true) - { - // 分割数据页，自增 - var sb = Builder.Clone(); - if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; - sb.Where += $"{name}>{Row}"; + // 分割数据页，自增 + var sb = Builder.Clone(); + if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; + sb.Where += $"{name}>{Row}"; - // 查询数据 - var dt = Dal.Query(sb, 0, BatchSize); - if (dt == null) break; + // 查询数据 + var dt = Dal.Query(sb, 0, BatchSize); + if (dt == null) break; - var count = dt.Rows.Count; - if (count == 0) break; + var count = dt.Rows.Count; + if (count == 0) break; - // 返回数据 - yield return dt; + // 返回数据 + yield return dt; - // 自增分割时，取最后一行 - Row = dt.Get(count - 1, field.ColumnName); + // 自增分割时，取最后一行 + Row = dt.Get(count - 1, field.ColumnName); - // 下一页 - TotalCount += count; - if (count < BatchSize) break; - } + // 下一页 + TotalCount += count; + if (count < BatchSize) break; } - #endregion } + #endregion } \ No newline at end of file diff --git a/XCode/Transform/PagingExtracter.cs b/XCode/Transform/PagingExtracter.cs index f4dbb18b7..6a8737ae0 100644 --- a/XCode/Transform/PagingExtracter.cs +++ b/XCode/Transform/PagingExtracter.cs @@ -1,82 +1,79 @@ -using System; -using System.Collections.Generic; -using NewLife.Data; +using NewLife.Data; using XCode.DataAccessLayer; -namespace XCode.Transform +namespace XCode.Transform; + +/// 分页数据抽取器 +/// +/// 通用抽取器，既没有Id序列也没有时间戳时使用。 +/// 采用分页技术抽取，通用性很强，但是随着页数增加，速度也会下降。 +/// +public class PagingExtracter : IExtracter { - /// 分页数据抽取器 - /// - /// 通用抽取器，既没有Id序列也没有时间戳时使用。 - /// 采用分页技术抽取，通用性很强，但是随着页数增加，速度也会下降。 - /// - public class PagingExtracter : IExtracter - { - #region 属性 - /// 数据层 - public DAL Dal { get; set; } + #region 属性 + /// 数据层 + public DAL Dal { get; set; } - /// 查询表达式 - public SelectBuilder Builder { get; set; } + /// 查询表达式 + public SelectBuilder Builder { get; set; } - /// 开始行。默认0 - public Int64 Row { get; set; } + /// 开始行。默认0 + public Int64 Row { get; set; } - /// 批大小。默认5000 - public Int32 BatchSize { get; set; } = 5000; - #endregion + /// 批大小。默认5000 + public Int32 BatchSize { get; set; } = 5000; + #endregion - #region 构造 - /// 实例化分页抽取器 - public PagingExtracter() { } + #region 构造 + /// 实例化分页抽取器 + public PagingExtracter() { } - /// 实例化分页抽取器 - /// - /// - public PagingExtracter(DAL dal, String tableName) - { - Dal = dal; - Builder = new SelectBuilder { Table = tableName }; - BatchSize = dal.Db.BatchSize; - if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; - } + /// 实例化分页抽取器 + /// + /// + public PagingExtracter(DAL dal, String tableName) + { + Dal = dal; + Builder = new SelectBuilder { Table = tableName }; + BatchSize = dal.Db.BatchSize; + if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; + } - /// 实例化分页抽取器 - /// - /// - /// - public PagingExtracter(DAL dal, String tableName, String orderBy) - { - Dal = dal; - Builder = new SelectBuilder { Table = tableName, OrderBy = orderBy }; - BatchSize = dal.Db.BatchSize; - if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; - } - #endregion + /// 实例化分页抽取器 + /// + /// + /// + public PagingExtracter(DAL dal, String tableName, String orderBy) + { + Dal = dal; + Builder = new SelectBuilder { Table = tableName, OrderBy = orderBy }; + BatchSize = dal.Db.BatchSize; + if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; + } + #endregion - #region 抽取数据 - /// 迭代抽取数据 - /// - public virtual IEnumerable Fetch() + #region 抽取数据 + /// 迭代抽取数据 + /// + public virtual IEnumerable Fetch() + { + while (true) { - while (true) - { - // 查询数据 - var dt = Dal.Query(Builder, Row, BatchSize); - if (dt == null) break; + // 查询数据 + var dt = Dal.Query(Builder, Row, BatchSize); + if (dt == null) break; - var count = dt.Rows.Count; - if (count == 0) break; + var count = dt.Rows.Count; + if (count == 0) break; - // 返回数据 - yield return dt; + // 返回数据 + yield return dt; - Row += BatchSize; + Row += BatchSize; - // 下一页 - if (count < BatchSize) break; - } + // 下一页 + if (count < BatchSize) break; } - #endregion } + #endregion } \ No newline at end of file diff --git a/XCode/Transform/TimeExtracter.cs b/XCode/Transform/TimeExtracter.cs index 3401ceb6a..3286a4414 100644 --- a/XCode/Transform/TimeExtracter.cs +++ b/XCode/Transform/TimeExtracter.cs @@ -1,119 +1,116 @@ -using System; -using System.Collections.Generic; -using NewLife; +using NewLife; using NewLife.Data; using XCode.DataAccessLayer; -namespace XCode.Transform +namespace XCode.Transform; + +/// 时间索引数据抽取器 +/// +/// 适用于带有时间索引字段的数据抽取器，速度飞快。 +/// +public class TimeExtracter : IExtracter { - /// 时间索引数据抽取器 - /// - /// 适用于带有时间索引字段的数据抽取器，速度飞快。 - /// - public class TimeExtracter : IExtracter - { - #region 属性 - /// 数据层 - public DAL Dal { get; set; } + #region 属性 + /// 数据层 + public DAL Dal { get; set; } - /// 查询表达式 - public SelectBuilder Builder { get; set; } + /// 查询表达式 + public SelectBuilder Builder { get; set; } - /// 时间字段 - public IDataColumn Field { get; set; } + /// 时间字段 + public IDataColumn Field { get; set; } - /// 开始行。分页时表示偏移行数，自增时表示下一个编号，默认0 - public Int64 Row { get; set; } + /// 开始行。分页时表示偏移行数，自增时表示下一个编号，默认0 + public Int64 Row { get; set; } - /// 开始抽取时间 - public DateTime StartTime { get; set; } + /// 开始抽取时间 + public DateTime StartTime { get; set; } - /// 批大小。默认5000 - public Int32 BatchSize { get; set; } = 5000; + /// 批大小。默认5000 + public Int32 BatchSize { get; set; } = 5000; - /// 总行数 - public Int32 TotalCount { get; set; } - #endregion + /// 总行数 + public Int32 TotalCount { get; set; } + #endregion - #region 构造 - /// 实例化自增抽取器 - public TimeExtracter() { } + #region 构造 + /// 实例化自增抽取器 + public TimeExtracter() { } - /// 实例化自增抽取器 - /// - /// - /// - public TimeExtracter(DAL dal, String tableName, IDataColumn field) - { - Dal = dal; - Builder = new SelectBuilder { Table = tableName, OrderBy = field.ColumnName + " asc" }; - Field = field; - BatchSize = dal.Db.BatchSize; - if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; - } - #endregion + /// 实例化自增抽取器 + /// + /// + /// + public TimeExtracter(DAL dal, String tableName, IDataColumn field) + { + Dal = dal; + Builder = new SelectBuilder { Table = tableName, OrderBy = field.ColumnName + " asc" }; + Field = field; + BatchSize = dal.Db.BatchSize; + if (BatchSize <= 0) BatchSize = XCodeSetting.Current.BatchSize; + } + #endregion - #region 抽取数据 - /// 迭代抽取数据 - /// - public virtual IEnumerable Fetch() + #region 抽取数据 + /// 迭代抽取数据 + /// + public virtual IEnumerable Fetch() + { + var field = Field; + var db = Dal.Db; + var name = db.FormatName(field); + while (true) { - var field = Field; - var db = Dal.Db; - var name = db.FormatName(field); - while (true) - { - // 分割数据页 - var sb = Builder.Clone(); - if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; - //修复拼接sql错误 - if (StartTime != null && StartTime > DateTime.MinValue) - sb.Where += $"{name}>{db.FormatValue(field, StartTime)}"; + // 分割数据页 + var sb = Builder.Clone(); + if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; + sb.Where += $"{name}>={db.FormatValue(field, StartTime)}"; - // 查询数据 - var dt = Dal.Query(sb, 0, BatchSize); - if (dt == null) break; + // 查询数据 + var dt = Dal.Query(sb, 0, BatchSize); + if (dt == null || dt.Rows == null) break; - var count = dt.Rows.Count; - if (count == 0) break; + var count = dt.Rows.Count; + if (count == 0) break; - // 返回数据 - yield return dt; + // 返回数据 + yield return dt; - // 分割时，取最后一行 - StartTime = dt.Get(count - 1, field.ColumnName); + // 分割时，取最后一行 + var time = dt.Get(count - 1, field.ColumnName); - // 如果满一页，则再查一次该时间，避免该时间的数据同时落入多页 - if (count == BatchSize) + // 如果满一页，则再查一次该时间，避免该时间的数据同时落入多页 + if (count == BatchSize) + { + sb = Builder.Clone(); + if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; + // 避免分页插入重复的一条数据 + sb.Where += $"{name}>={db.FormatValue(field, time)} And {name}<{db.FormatValue(field, time.AddSeconds(1))}"; + + // 查询数据，该时间点数据也可能有多页 + var startRow = 0; + while (true) { - sb = Builder.Clone(); - if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; - //避免分页插入重复的一条数据 - sb.Where += $"{name}>{db.FormatValue(field, StartTime)} And {name}<{db.FormatValue(field, StartTime.AddSeconds(1))}"; - - // 查询数据，该时间点数据也可能有多页 - var startRow = 0; - while (true) - { - dt = Dal.Query(sb, startRow, BatchSize); - if (dt == null) break; - - var count2 = dt.Rows.Count; - if (count2 == 0) break; - - yield return dt; - - startRow += count2; - count += count2; - if (count2 < BatchSize) break; - } - } + dt = Dal.Query(sb, startRow, BatchSize); + if (dt == null || dt.Rows == null) break; + + var count2 = dt.Rows.Count; + if (count2 == 0) break; - // 下一页 - TotalCount += count; - if (count < BatchSize) break; + yield return dt; + + startRow += count2; + count += count2; + if (count2 < BatchSize) break; + } } + + StartTime = time.AddSeconds(1); + + // 下一页 + TotalCount += count; + if (count < BatchSize) break; } - #endregion } + #endregion } \ No newline at end of file From 6242f37aa202035d5bada430d6d28614fb5bc310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=99=BA=E8=83=BD=E5=A4=A7=E7=9F=B3=E5=A4=B4?= Date: Wed, 28 Feb 2024 00:00:35 +0800 Subject: [PATCH 2/6] =?UTF-8?q?[refactor]=20=E9=87=8D=E6=9E=84=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=8A=BD=E5=8F=96=E5=99=A8=EF=BC=8C=E7=AE=80=E5=8C=96?= =?UTF-8?q?=E7=9B=B8=E5=90=8C=E6=97=B6=E9=97=B4=E8=B7=A8=E9=A1=B5=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E6=8C=89=E6=97=B6=E9=97=B4=E5=88=86=E7=89=87?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E3=80=82=E5=A6=82=E6=9E=9C=E6=9C=89=E5=A4=9A?= =?UTF-8?q?=E9=A1=B5=EF=BC=8C=E5=88=99=E5=88=86=E7=89=87=E5=86=85=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E5=88=86=E9=A1=B5=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- XCode/DataAccessLayer/DbPackage.cs | 4 +- XCode/Transform/IExtracter.cs | 7 ++- XCode/Transform/PagingExtracter.cs | 3 + XCode/Transform/TimeExtracter.cs | 93 ++++++++++++++++-------------- 4 files changed, 61 insertions(+), 46 deletions(-) diff --git a/XCode/DataAccessLayer/DbPackage.cs b/XCode/DataAccessLayer/DbPackage.cs index 347ae180c..bf40e95bc 100644 --- a/XCode/DataAccessLayer/DbPackage.cs +++ b/XCode/DataAccessLayer/DbPackage.cs @@ -93,7 +93,7 @@ public virtual Int32 Backup(IDataTable table, Stream stream) { foreach (var dt in extracer.Fetch()) { - var row = extracer.Row; + var row = extracer.TotalCount; var count = dt.Rows.Count; WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, connName, row, count); if (count == 0) break; @@ -528,7 +528,7 @@ public virtual Int32 Sync(IDataTable table, String connName, Boolean syncSchema foreach (var dt in extracer.Fetch()) { - var row = extracer.Row; + var row = extracer.TotalCount; var count = dt.Rows.Count; WriteLog("同步[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, Dal.ConnName, row, count); diff --git a/XCode/Transform/IExtracter.cs b/XCode/Transform/IExtracter.cs index a2c926e25..a468ee3d6 100644 --- a/XCode/Transform/IExtracter.cs +++ b/XCode/Transform/IExtracter.cs @@ -9,8 +9,11 @@ namespace XCode.Transform; public interface IExtracter { #region 属性 - /// 开始行。分页时表示偏移行数，自增时表示下一个编号，默认0 - Int64 Row { get; set; } + ///// 开始行。分页时表示偏移行数，自增时表示下一个编号，默认0 + //Int64 Row { get; set; } + + /// 总行数 + Int32 TotalCount { get; set; } #endregion #region 抽取数据 diff --git a/XCode/Transform/PagingExtracter.cs b/XCode/Transform/PagingExtracter.cs index 6a8737ae0..cf1002308 100644 --- a/XCode/Transform/PagingExtracter.cs +++ b/XCode/Transform/PagingExtracter.cs @@ -22,6 +22,9 @@ public class PagingExtracter : IExtracter /// 批大小。默认5000 public Int32 BatchSize { get; set; } = 5000; + + /// 总行数 + public Int32 TotalCount { get; set; } #endregion #region 构造 diff --git a/XCode/Transform/TimeExtracter.cs b/XCode/Transform/TimeExtracter.cs index 3286a4414..b6960b82e 100644 --- a/XCode/Transform/TimeExtracter.cs +++ b/XCode/Transform/TimeExtracter.cs @@ -1,5 +1,4 @@ -using NewLife; -using NewLife.Data; +using NewLife.Data; using XCode.DataAccessLayer; namespace XCode.Transform; @@ -53,63 +52,73 @@ public TimeExtracter(DAL dal, String tableName, IDataColumn field) #region 抽取数据 /// 迭代抽取数据 + /// + /// 从StartTime开始，分区加分页抽取数据。每次抽取完成后，将StartTime设置为最后一行的时间，然后加上1秒作为下一次的开始时间。 + /// 逼近当前时间时停止，下次再调用Fetch时，将从上次停止的地方继续抽取。 + /// /// public virtual IEnumerable Fetch() { var field = Field; var db = Dal.Db; var name = db.FormatName(field); - while (true) + + // 第一次查询，不带时间条件，目的是为了找到第一个时间 + if (StartTime.Year < 2000) { - // 分割数据页 + // 查询数据 var sb = Builder.Clone(); - if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; - sb.Where += $"{name}>={db.FormatValue(field, StartTime)}"; + var dt = Dal.Query(sb, 0, 1); - // 查询数据 - var dt = Dal.Query(sb, 0, BatchSize); - if (dt == null || dt.Rows == null) break; + // 第一页都没有数据，直接返回 + if (dt == null || dt.Rows == null || dt.Rows.Count <= 0) + yield break; + + StartTime = dt.Get(0, field.ColumnName); - var count = dt.Rows.Count; - if (count == 0) break; + if (StartTime.Year < 2000) yield break; + } - // 返回数据 - yield return dt; + // 时间步进，分页查询 + var minStep = TimeSpan.FromSeconds(60); + var maxStep = TimeSpan.FromDays(1); + var step = minStep; + while (true) + { + var now = DateTime.Now; + var end = StartTime.Add(step); + if (end > now) end = now; - // 分割时，取最后一行 - var time = dt.Get(count - 1, field.ColumnName); + // 按时间分片查询。如果有多页，则分片内使用分页查询 + var sb = Builder.Clone().AppendWhereAnd($"{name}>={db.FormatValue(field, StartTime)} And {name}<{db.FormatValue(field, end)}"); - // 如果满一页，则再查一次该时间，避免该时间的数据同时落入多页 - if (count == BatchSize) + var startRow = 0; + while (true) { - sb = Builder.Clone(); - if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; - // 避免分页插入重复的一条数据 - sb.Where += $"{name}>={db.FormatValue(field, time)} And {name}<{db.FormatValue(field, time.AddSeconds(1))}"; - - // 查询数据，该时间点数据也可能有多页 - var startRow = 0; - while (true) - { - dt = Dal.Query(sb, startRow, BatchSize); - if (dt == null || dt.Rows == null) break; - - var count2 = dt.Rows.Count; - if (count2 == 0) break; - - yield return dt; - - startRow += count2; - count += count2; - if (count2 < BatchSize) break; - } + // 分页查询数据 + var dt = Dal.Query(sb, startRow, BatchSize); + if (dt == null || dt.Rows == null) break; + + var count = dt.Rows.Count; + if (count == 0) break; + + // 返回数据 + yield return dt; + + // 取最后一行的时间，加上1秒作为下一次分片的开始时间。多次赋值，最后一页为准 + StartTime = dt.Get(count - 1, field.ColumnName).AddSeconds(1); + + startRow += count; + TotalCount += count; + if (count == BatchSize) break; } - StartTime = time.AddSeconds(1); + // 踏空，加大步进 + if (startRow == 0 && step < maxStep) + step += step; - // 下一页 - TotalCount += count; - if (count < BatchSize) break; + // 止步于当前时间 + if (StartTime.Add(minStep) >= DateTime.Now) break; } } #endregion From ac47e8186b82c9579507ceb3755a9a532ac5d111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=99=BA=E8=83=BD=E5=A4=A7=E7=9F=B3=E5=A4=B4?= Date: Wed, 28 Feb 2024 00:12:29 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E9=9D=9E=E8=87=AA=E5=A2=9E=E7=9A=84?= =?UTF-8?q?=E6=95=B4=E6=95=B0=E4=B8=BB=E9=94=AE=EF=BC=8C=E4=B9=9F=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E4=BD=BF=E7=94=A8Id=E6=8A=BD=E5=8F=96=E5=99=A8?= =?UTF-8?q?=EF=BC=8C=E4=BE=8B=E5=A6=82=E9=9B=AA=E8=8A=B1Id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- XCode/DataAccessLayer/DbPackage.cs | 6 +++--- XCode/Transform/IdExtracter.cs | 17 +++++++---------- XCode/Transform/PagingExtracter.cs | 6 +++--- XCode/Transform/TimeExtracter.cs | 4 ++-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/XCode/DataAccessLayer/DbPackage.cs b/XCode/DataAccessLayer/DbPackage.cs index bf40e95bc..4dcab8a51 100644 --- a/XCode/DataAccessLayer/DbPackage.cs +++ b/XCode/DataAccessLayer/DbPackage.cs @@ -161,8 +161,8 @@ protected virtual IExtracter GetExtracter(IDataTable table) var id = table.Columns.FirstOrDefault(e => e.Identity); if (id == null) { - var pks = table.Columns.Where(e => e.PrimaryKey).ToList(); - if (pks.Count == 1 && pks[0].DataType.IsInt()) id = pks[0]; + var pks = table.PrimaryKeys; + if (pks.Length >= 1 && pks[0].DataType.IsInt()) id = pks[0]; } if (id != null) return new IdExtracter(Dal, tableName, id); @@ -198,7 +198,7 @@ protected virtual IExtracter GetExtracter(IDataTable table) // 默认第一个字段 var dc = table.Columns.FirstOrDefault(); - return dc != null ? new PagingExtracter(Dal, tableName, dc.ColumnName) : (IExtracter)new PagingExtracter(Dal, tableName); + return new PagingExtracter(Dal, tableName, dc?.ColumnName); } /// 备份单表数据到文件 diff --git a/XCode/Transform/IdExtracter.cs b/XCode/Transform/IdExtracter.cs index a316a61ec..1d52181f3 100644 --- a/XCode/Transform/IdExtracter.cs +++ b/XCode/Transform/IdExtracter.cs @@ -1,12 +1,11 @@ -using NewLife; -using NewLife.Data; +using NewLife.Data; using XCode.DataAccessLayer; namespace XCode.Transform; -/// 整型主键数据抽取器 +/// 整数主键数据抽取器 /// -/// 适用于带有自增字段或雪花Id字段的数据抽取器，速度飞快。 +/// 适用于带有自增字段或雪花Id字段的数据抽取器，每次只抽取一页，采取主键步进，速度飞快。 /// public class IdExtracter : IExtracter { @@ -31,10 +30,10 @@ public class IdExtracter : IExtracter #endregion #region 构造 - /// 实例化自增抽取器 + /// 实例化整数主键数据抽取器 public IdExtracter() { } - /// 实例化自增抽取器 + /// 实例化整数主键数据抽取器 /// /// /// @@ -59,13 +58,11 @@ public virtual IEnumerable Fetch() while (true) { // 分割数据页，自增 - var sb = Builder.Clone(); - if (!sb.Where.IsNullOrEmpty()) sb.Where += " And "; - sb.Where += $"{name}>{Row}"; + var sb = Builder.Clone().AppendWhereAnd($"{name}>{Row}"); // 查询数据 var dt = Dal.Query(sb, 0, BatchSize); - if (dt == null) break; + if (dt == null || dt.Rows == null) break; var count = dt.Rows.Count; if (count == 0) break; diff --git a/XCode/Transform/PagingExtracter.cs b/XCode/Transform/PagingExtracter.cs index cf1002308..637c42fd1 100644 --- a/XCode/Transform/PagingExtracter.cs +++ b/XCode/Transform/PagingExtracter.cs @@ -28,7 +28,7 @@ public class PagingExtracter : IExtracter #endregion #region 构造 - /// 实例化分页抽取器 + /// 实例化分页数据抽取器 public PagingExtracter() { } /// 实例化分页抽取器 @@ -46,7 +46,7 @@ public PagingExtracter(DAL dal, String tableName) /// /// /// - public PagingExtracter(DAL dal, String tableName, String orderBy) + public PagingExtracter(DAL dal, String tableName, String? orderBy) { Dal = dal; Builder = new SelectBuilder { Table = tableName, OrderBy = orderBy }; @@ -64,7 +64,7 @@ public virtual IEnumerable Fetch() { // 查询数据 var dt = Dal.Query(Builder, Row, BatchSize); - if (dt == null) break; + if (dt == null || dt.Rows == null) break; var count = dt.Rows.Count; if (count == 0) break; diff --git a/XCode/Transform/TimeExtracter.cs b/XCode/Transform/TimeExtracter.cs index b6960b82e..f870e8cbb 100644 --- a/XCode/Transform/TimeExtracter.cs +++ b/XCode/Transform/TimeExtracter.cs @@ -33,10 +33,10 @@ public class TimeExtracter : IExtracter #endregion #region 构造 - /// 实例化自增抽取器 + /// 实例化时间索引数据抽取器 public TimeExtracter() { } - /// 实例化自增抽取器 + /// 实例化时间索引数据抽取器 /// /// /// From fb74b37a1693a2da8cb513bf35fa541dea79c67e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=A7=E7=9F=B3=E5=A4=B4?= Date: Wed, 28 Feb 2024 15:56:27 +0800 Subject: [PATCH 4/6] =?UTF-8?q?[fix]=20=E4=BF=AE=E6=AD=A3=E4=B8=8A?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E5=AF=B9SqlServer=E7=9A=84=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=AF=BC=E8=87=B4DbMetaData=E6=8A=A5=E9=94=99=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- XCode/DataAccessLayer/Database/SqlServer.cs | 20 ++++++++++++++-- .../MetaData/DbMetaData_Positive.cs | 24 ++++--------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/XCode/DataAccessLayer/Database/SqlServer.cs b/XCode/DataAccessLayer/Database/SqlServer.cs index fb4f3bb9a..55948f182 100644 --- a/XCode/DataAccessLayer/Database/SqlServer.cs +++ b/XCode/DataAccessLayer/Database/SqlServer.cs @@ -978,11 +978,18 @@ public override IList GetTableNames() return list; } - private DataTable AllFields = null; - private DataTable AllIndexes = null; + private DataTable? AllFields = null; + private DataTable? AllIndexes = null; protected override void FixField(IDataColumn field, DataRow dr) { + //修复sqlserver同步到sqlserver建表长度为1的bug + var rawType = field.RawType?.ToLower(); + if (rawType == "nvarchar" || rawType == "varchar" || rawType == "char" || rawType == "binary") + field.RawType += "(" + field.Length + ")"; + if (rawType == "varbinary" && field.Length == -1) + field.RawType = "varbinary(max)"; + base.FixField(field, dr); var rows = AllFields?.Select("表名='" + field.Table.TableName + "' And 字段名='" + field.ColumnName + "'", null); @@ -1081,6 +1088,15 @@ public override String FieldClause(IDataColumn field, Boolean onlyDefine) // if (field.DataType == typeof(String) && pi == "-1" && IsSQL2005) return "MAX"; // return pi; //} + + protected override String? GetFieldType(IDataColumn field) + { + //mysql 转sqlserver + if (field.RawType?.ToLower() == "longblob") + return "varbinary(max)"; + + return base.GetFieldType(field); + } #endregion #region 取得字段信息的SQL模版 diff --git a/XCode/DataAccessLayer/MetaData/DbMetaData_Positive.cs b/XCode/DataAccessLayer/MetaData/DbMetaData_Positive.cs index 6073b794d..d58921aaf 100644 --- a/XCode/DataAccessLayer/MetaData/DbMetaData_Positive.cs +++ b/XCode/DataAccessLayer/MetaData/DbMetaData_Positive.cs @@ -1,11 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Data; +using System.Data; using System.Data.Common; -using System.Linq; using NewLife; -using NewLife.Log; -using NewLife.Serialization; using XCode.Common; using XCode.Exceptions; @@ -212,17 +207,9 @@ protected virtual List GetFields(IDataTable table, DataRow[] rows) // 原始数据类型 field.RawType = GetDataRowValue(dr, "DATA_TYPE", "DATATYPE", "COLUMN_DATA_TYPE"); - // 长度 field.Length = GetDataRowValue(dr, "CHARACTER_MAXIMUM_LENGTH", "LENGTH", "COLUMN_SIZE"); - //修复sqlserver同步到sqlserver建表长度为1的bug - var rawType = field.RawType.ToLower(); - if (rawType == "nvarchar" || rawType == "varchar" || rawType == "char" || rawType == "binary") - field.RawType += "(" + field.Length + ")"; - if (rawType == "varbinary" && field.Length == -1) - field.RawType = "varbinary(max)"; - if (field is XField fi) { // 精度 与 位数 @@ -231,7 +218,6 @@ protected virtual List GetFields(IDataTable table, DataRow[] rows) if (field.Length == 0) field.Length = fi.Precision; } - // 允许空 if (TryGetDataRowValue(dr, "IS_NULLABLE", out b)) field.Nullable = b; @@ -344,7 +330,7 @@ protected virtual void FixIndex(IDataIndex index, DataRow dr) { } protected IDictionary Types { get; set; } = null!; protected List>? _FieldTypeMaps; - /// 字段类型映射 + /// 字段类型映射（数据库-实体类） protected virtual List> FieldTypeMaps { get @@ -359,6 +345,8 @@ protected virtual List> FieldTypeMaps // 因为等价，字节需要能够互相映射 new(typeof(Byte), typeof(SByte)), new(typeof(Byte), typeof(Boolean)), + new(typeof(Boolean), typeof(Byte)), + new(typeof(Byte), typeof(Int32)), new(typeof(UInt16), typeof(Int16)), new(typeof(Int16), typeof(UInt16)), @@ -377,6 +365,7 @@ protected virtual List> FieldTypeMaps new(typeof(Int64), typeof(UInt64)), //list.Add(new KeyValuePair(typeof(UInt64), typeof(Int32))); //list.Add(new KeyValuePair(typeof(Int64), typeof(Int32))); + new(typeof(Int64), typeof(Int32)), // 数据库使用Double，实体类使用Decimal new(typeof(Double), typeof(Decimal)), @@ -419,9 +408,6 @@ protected virtual List> FieldTypeMaps var typeName = ns.FirstOrDefault(); // 大文本选第二个类型 if (ns.Length > 1 && type == typeof(String) && (field.Length <= 0 || field.Length >= Database.LongTextLength)) typeName = ns[1]; - //mysql 转sqlserver - if (field.RawType.ToLower() == "longblob") - typeName = "varbinary(max)"; if (typeName.Contains("{0}")) { if (typeName.Contains("{1}")) From 0bd94e56aa796a8919b02b4b85f782fbeb2a5fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=A7=E7=9F=B3=E5=A4=B4?= Date: Wed, 28 Feb 2024 16:56:13 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E7=A1=AE=E4=BF=9DCreateFactory=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E5=AF=B9=E8=B1=A1=E5=94=AF=E4=B8=80=EF=BC=8C=E5=90=A6?= =?UTF-8?q?=E5=88=99=E5=BD=B1=E5=93=8DSnowflake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- XCode/Entity/EntityFactory.cs | 471 ++++++++++++------------ XCode/Entity/Entity_Meta.cs | 6 +- XCode/Entity/Entity_Operate.cs | 40 +- XCode/Entity/IEntityOperate.cs | 22 +- XUnitTest.XCode/Membership/AreaTests.cs | 2 +- 5 files changed, 269 insertions(+), 272 deletions(-) diff --git a/XCode/Entity/EntityFactory.cs b/XCode/Entity/EntityFactory.cs index 0a7d5f154..55064447e 100644 --- a/XCode/Entity/EntityFactory.cs +++ b/XCode/Entity/EntityFactory.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; +using System.Collections.Concurrent; using System.Reflection; using NewLife; using NewLife.Log; @@ -9,299 +6,299 @@ using XCode.Configuration; using XCode.DataAccessLayer; -namespace XCode +namespace XCode; + +/// 实体工厂 +public static class EntityFactory { - /// 实体工厂 - public static class EntityFactory + #region 创建实体工厂 + private static readonly ConcurrentDictionary _factories = new(); + /// 实体工厂集合 + public static IDictionary Entities => _factories; + + /// 创建实体操作接口 + /// + /// 因为只用来做实体操作，所以只需要一个实例即可。 + /// 调用平均耗时3.95ns，57.39%在EnsureInit + /// + /// 类型 + /// + public static IEntityFactory CreateFactory(Type type) { - #region 创建实体工厂 - private static readonly ConcurrentDictionary _factories = new(); - /// 实体工厂集合 - public static IDictionary Entities => _factories; - - /// 创建实体操作接口 - /// - /// 因为只用来做实体操作，所以只需要一个实例即可。 - /// 调用平均耗时3.95ns，57.39%在EnsureInit - /// - /// 类型 - /// - public static IEntityFactory CreateFactory(Type type) - { - if (type == null) throw new ArgumentNullException(nameof(type)); + if (type == null) throw new ArgumentNullException(nameof(type)); - if (_factories.TryGetValue(type, out var factory)) return factory; + if (_factories.TryGetValue(type, out var factory)) return factory; - // 有可能有子类直接继承实体类，这里需要找到继承泛型实体类的那一层 - while (!type.BaseType.IsGenericType) type = type.BaseType; + // 有可能有子类直接继承实体类，这里需要找到继承泛型实体类的那一层 + while (!type.BaseType.IsGenericType) type = type.BaseType; - if (_factories.TryGetValue(type, out factory)) return factory; + if (_factories.TryGetValue(type, out factory)) return factory; - //// 确保实体类已被初始化，实际上，因为实体类静态构造函数中会注册IEntityFactory，所以下面的委托按理应该再也不会被执行了 - //// 先实例化，在锁里面添加到列表但不实例化，避免实体类的实例化过程中访问CreateFactory导致死锁产生 - //type.CreateInstance(); + //// 确保实体类已被初始化，实际上，因为实体类静态构造函数中会注册IEntityFactory，所以下面的委托按理应该再也不会被执行了 + //// 先实例化，在锁里面添加到列表但不实例化，避免实体类的实例化过程中访问CreateFactory导致死锁产生 + //type.CreateInstance(); - //if (!_factories.TryGetValue(type, out factory)) throw new XCodeException("无法创建[{0}]的实体工厂！", type.FullName); + //if (!_factories.TryGetValue(type, out factory)) throw new XCodeException("无法创建[{0}]的实体工厂！", type.FullName); - // 读取特性中指定的自定义工程，若未指定，则使用默认工厂 - var att = type.GetCustomAttribute(); - var factoryType = att?.Type; - if (factoryType == null) factoryType = typeof(Entity<>).MakeGenericType(type).GetNestedType("DefaultEntityFactory").MakeGenericType(type); + // 读取特性中指定的自定义工程，若未指定，则使用默认工厂 + var att = type.GetCustomAttribute(); + var factoryType = att?.Type; + //factoryType ??= typeof(Entity<>).MakeGenericType(type).GetNestedType("DefaultEntityFactory").MakeGenericType(type); + factoryType ??= typeof(Entity<>.DefaultEntityFactory).MakeGenericType(type); - factory = factoryType?.CreateInstance() as IEntityFactory; - if (factory == null) throw new XCodeException("无法创建[{0}]的实体工厂！", type.FullName); + factory = factoryType?.CreateInstance() as IEntityFactory; + if (factory == null) throw new XCodeException("无法创建[{0}]的实体工厂！", type.FullName); - //!!! 有可能多线程同时初始化相同实体类的实体工厂，需要避免返回不同的工厂实例，确保工厂成员的唯一性，例如雪花实例 - //_factories.TryAdd(type, factory); - factory = _factories.GetOrAdd(type, factory); + //!!! 有可能多线程同时初始化相同实体类的实体工厂，需要避免返回不同的工厂实例，确保工厂成员的唯一性，例如雪花实例 + //_factories.TryAdd(type, factory); + factory = _factories.GetOrAdd(type, factory); - return factory; - } + return factory; + } - /// 根据类型创建实体工厂 - /// - /// - public static IEntityFactory AsFactory(this Type type) => CreateFactory(type); + /// 根据类型创建实体工厂 + /// + /// + public static IEntityFactory AsFactory(this Type type) => CreateFactory(type); - /// 使用指定的实体对象创建实体操作接口，主要用于Entity内部调用，避免反射带来的损耗 - /// 类型 - /// - /// - public static IEntityFactory Register(Type type, IEntityFactory factory) - { - _factories[type] = factory ?? throw new ArgumentNullException(nameof(factory)); + /// 使用指定的实体对象创建实体操作接口，主要用于Entity内部调用，避免反射带来的损耗 + /// 类型 + /// + /// + public static IEntityFactory Register(Type type, IEntityFactory factory) + { + _factories[type] = factory ?? throw new ArgumentNullException(nameof(factory)); - return factory; - } - #endregion - - #region 加载插件 - ///// 列出所有实体类 - ///// - //public static List LoadEntities() - //{ - // return typeof(IEntity).GetAllSubclasses().ToList(); - //} - - /// 获取指定连接名下的所有实体类 - /// - /// - public static IEnumerable LoadEntities(String connName) + return factory; + } + #endregion + + #region 加载插件 + ///// 列出所有实体类 + ///// + //public static List LoadEntities() + //{ + // return typeof(IEntity).GetAllSubclasses().ToList(); + //} + + /// 获取指定连接名下的所有实体类 + /// + /// + public static IEnumerable LoadEntities(String connName) + { + foreach (var item in typeof(IEntity).GetAllSubclasses()) { - foreach (var item in typeof(IEntity).GetAllSubclasses()) - { - // 实体类的基类必须是泛型，避免多级继承导致误判 - if (!item.BaseType.IsGenericType) continue; - - var ti = TableItem.Create(item); - if (ti == null) - XTrace.WriteLine("实体类[{0}]无法创建TableItem", item.FullName); - else if (ti.ConnName == connName) - yield return item; - } + // 实体类的基类必须是泛型，避免多级继承导致误判 + if (!item.BaseType.IsGenericType) continue; + + var ti = TableItem.Create(item); + if (ti == null) + XTrace.WriteLine("实体类[{0}]无法创建TableItem", item.FullName); + else if (ti.ConnName == connName) + yield return item; } + } - /// 获取指定连接名下的初始化时检查的所有实体数据表，用于反向工程检查表架构 - /// - /// - /// - public static List GetTables(String connName, Boolean checkMode) + /// 获取指定连接名下的初始化时检查的所有实体数据表，用于反向工程检查表架构 + /// + /// + /// + public static List GetTables(String connName, Boolean checkMode) + { + var tables = new List(); + // 记录每个表名对应的实体类 + var dic = new Dictionary(StringComparer.OrdinalIgnoreCase); + var list = new List(); + var list2 = new List(); + foreach (var item in LoadEntities(connName)) { - var tables = new List(); - // 记录每个表名对应的实体类 - var dic = new Dictionary(StringComparer.OrdinalIgnoreCase); - var list = new List(); - var list2 = new List(); - foreach (var item in LoadEntities(connName)) - { - list.Add(item.Name); + list.Add(item.Name); - // 过滤掉第一次使用才加载的 - if (checkMode) - { - var att = item.GetCustomAttribute(true); - if (att != null && att.Mode != ModelCheckModes.CheckAllTablesWhenInit) continue; - } - list2.Add(item.Name); - - var table = TableItem.Create(item).DataTable; + // 过滤掉第一次使用才加载的 + if (checkMode) + { + var att = item.GetCustomAttribute(true); + if (att != null && att.Mode != ModelCheckModes.CheckAllTablesWhenInit) continue; + } + list2.Add(item.Name); - // 判断表名是否已存在 - if (dic.TryGetValue(table.TableName, out var type)) - { - // 两个都不是，报错吧！ - var msg = $"设计错误！发现表{table.TableName}同时被两个实体类（{type.FullName}和{item.FullName}）使用！"; - XTrace.WriteLine(msg); - throw new XCodeException(msg); - } - else - { - dic.Add(table.TableName, item); - } + var table = TableItem.Create(item).DataTable; - tables.Add(table); + // 判断表名是否已存在 + if (dic.TryGetValue(table.TableName, out var type)) + { + // 两个都不是，报错吧！ + var msg = $"设计错误！发现表{table.TableName}同时被两个实体类（{type.FullName}和{item.FullName}）使用！"; + XTrace.WriteLine(msg); + throw new XCodeException(msg); } - - if (DAL.Debug) + else { - DAL.WriteLog("[{0}]的所有实体类（{1}个）：{2}", connName, list.Count, String.Join(",", list.ToArray())); - DAL.WriteLog("[{0}]需要检查架构的实体类（{1}个）：{2}", connName, list2.Count, String.Join(",", list2.ToArray())); + dic.Add(table.TableName, item); } - return tables; + tables.Add(table); } - #endregion - #region 现代化用法 - /// 初始化所有数据库连接的实体类和数据表 - public static void InitAll() + if (DAL.Debug) { - using var span = DefaultTracer.Instance?.NewSpan("db:InitAll"); - try - { - DAL.WriteLog("初始化所有数据库连接的实体类和数据表"); + DAL.WriteLog("[{0}]的所有实体类（{1}个）：{2}", connName, list.Count, String.Join(",", list.ToArray())); + DAL.WriteLog("[{0}]需要检查架构的实体类（{1}个）：{2}", connName, list2.Count, String.Join(",", list2.ToArray())); + } - // 加载所有实体类 - var types = typeof(IEntity).GetAllSubclasses().Where(e => e.BaseType.IsGenericType).ToList(); - var connNames = new List(); - foreach (var item in types) - { - var ti = TableItem.Create(item); - if (ti == null || connNames.Contains(ti.ConnName) || ti.ModelCheckMode != ModelCheckModes.CheckAllTablesWhenInit) continue; - connNames.Add(ti.ConnName); + return tables; + } + #endregion - Init(ti.ConnName, types, true); - } - } - catch (Exception ex) + #region 现代化用法 + /// 初始化所有数据库连接的实体类和数据表 + public static void InitAll() + { + using var span = DefaultTracer.Instance?.NewSpan("db:InitAll"); + try + { + DAL.WriteLog("初始化所有数据库连接的实体类和数据表"); + + // 加载所有实体类 + var types = typeof(IEntity).GetAllSubclasses().Where(e => e.BaseType.IsGenericType).ToList(); + var connNames = new List(); + foreach (var item in types) { - span?.SetError(ex, null); - throw; + var ti = TableItem.Create(item); + if (ti == null || connNames.Contains(ti.ConnName) || ti.ModelCheckMode != ModelCheckModes.CheckAllTablesWhenInit) continue; + connNames.Add(ti.ConnName); + + Init(ti.ConnName, types, true); } } - - /// 初始化所有数据库连接的实体类和数据表 - public static async Task InitAllAsync() + catch (Exception ex) { - using var span = DefaultTracer.Instance?.NewSpan("db:InitAll"); - try - { - DAL.WriteLog("异步初始化所有数据库连接的实体类和数据表"); - - var ts = new List(); + span?.SetError(ex, null); + throw; + } + } - // 加载所有实体类 - var types = typeof(IEntity).GetAllSubclasses().Where(e => e.BaseType.IsGenericType).ToList(); - var connNames = new List(); - foreach (var item in types) - { - var ti = TableItem.Create(item); - if (ti == null || connNames.Contains(ti.ConnName) || ti.ModelCheckMode != ModelCheckModes.CheckAllTablesWhenInit) continue; - connNames.Add(ti.ConnName); + /// 初始化所有数据库连接的实体类和数据表 + public static async Task InitAllAsync() + { + using var span = DefaultTracer.Instance?.NewSpan("db:InitAll"); + try + { + DAL.WriteLog("异步初始化所有数据库连接的实体类和数据表"); - ts.Add(Task.Run(() => Init(ti.ConnName, types, false))); - } + var ts = new List(); - await Task.WhenAll(ts); - } - catch (Exception ex) + // 加载所有实体类 + var types = typeof(IEntity).GetAllSubclasses().Where(e => e.BaseType.IsGenericType).ToList(); + var connNames = new List(); + foreach (var item in types) { - span?.SetError(ex, null); - throw; + var ti = TableItem.Create(item); + if (ti == null || connNames.Contains(ti.ConnName) || ti.ModelCheckMode != ModelCheckModes.CheckAllTablesWhenInit) continue; + connNames.Add(ti.ConnName); + + ts.Add(Task.Run(() => Init(ti.ConnName, types, false))); } + + await Task.WhenAll(ts); + } + catch (Exception ex) + { + span?.SetError(ex, null); + throw; } + } - /// 初始化指定连接，执行反向工程检查，初始化数据 - /// 连接名 - public static void InitConnection(String connName) => Init(connName, null, true); + /// 初始化指定连接，执行反向工程检查，初始化数据 + /// 连接名 + public static void InitConnection(String connName) => Init(connName, null, true); - private static void Init(String connName, IList? types, Boolean throwOnError) + private static void Init(String connName, IList? types, Boolean throwOnError) + { + using var span = DefaultTracer.Instance?.NewSpan($"db:{connName}:InitConnection", connName); + try { - using var span = DefaultTracer.Instance?.NewSpan($"db:{connName}:InitConnection", connName); - try - { - // 加载所有实体类 - types ??= typeof(IEntity).GetAllSubclasses().Where(e => e.BaseType.IsGenericType).ToList(); + // 加载所有实体类 + types ??= typeof(IEntity).GetAllSubclasses().Where(e => e.BaseType.IsGenericType).ToList(); - // 初始化工厂 - var facts = new List(); - foreach (var type in types) + // 初始化工厂 + var facts = new List(); + foreach (var type in types) + { + var ti = TableItem.Create(type); + if (ti != null && ti.ConnName == connName && ti.ModelCheckMode == ModelCheckModes.CheckAllTablesWhenInit) { - var ti = TableItem.Create(type); - if (ti != null && ti.ConnName == connName && ti.ModelCheckMode == ModelCheckModes.CheckAllTablesWhenInit) - { - var fact = CreateFactory(type); - facts.Add(fact); - } + var fact = CreateFactory(type); + facts.Add(fact); } + } - var dal = DAL.Create(connName); - DAL.WriteLog("初始化数据库：{0}/{1} 实体类[{3}]：{2}", connName, dal.DbType, facts.Join(",", e => e.EntityType.Name), facts.Count); + var dal = DAL.Create(connName); + DAL.WriteLog("初始化数据库：{0}/{1} 实体类[{3}]：{2}", connName, dal.DbType, facts.Join(",", e => e.EntityType.Name), facts.Count); - // 反向工程检查 - if (dal.Db.Migration > Migration.Off) + // 反向工程检查 + if (dal.Db.Migration > Migration.Off) + { + //var tables = facts.Select(e => e.Table.DataTable).ToArray(); + var tables = new List(); + foreach (var item in facts) { - //var tables = facts.Select(e => e.Table.DataTable).ToArray(); - var tables = new List(); - foreach (var item in facts) + // 克隆一份，防止修改 + var table = item.Table.DataTable; + table = (table.Clone() as IDataTable)!; + + if (/*table != null &&*/ table.TableName != item.TableName) { - // 克隆一份，防止修改 - var table = item.Table.DataTable; - table = (table.Clone() as IDataTable)!; - - if (/*table != null &&*/ table.TableName != item.TableName) - { - // 表名去掉前缀 - var name = item.TableName; - if (name.Contains(".")) name = name.Substring("."); - - table.TableName = name; - } - tables.Add(table); - dal.CheckAndAdd(table.TableName); - } - dal.SetTables(tables.ToArray()); - } + // 表名去掉前缀 + var name = item.TableName; + if (name.Contains(".")) name = name.Substring("."); - // 实体类初始化数据 - foreach (var item in facts) - { - //if (item.Default is EntityBase entity) - //{ - // entity.InitData(); - //} - item.Session.InitData(); + table.TableName = name; + } + tables.Add(table); + dal.CheckAndAdd(table.TableName); } + dal.SetTables(tables.ToArray()); } - catch (Exception ex) - { - span?.SetError(ex, null); - if (throwOnError) throw; + // 实体类初始化数据 + foreach (var item in facts) + { + //if (item.Default is EntityBase entity) + //{ + // entity.InitData(); + //} + item.Session.InitData(); } } - - /// 初始化指定实体类，执行反向工程检查，初始化数据 - /// - public static void InitEntity(Type entityType) + catch (Exception ex) { - if (entityType == null) throw new ArgumentNullException(nameof(entityType)); + span?.SetError(ex, null); - using var span = DefaultTracer.Instance?.NewSpan("db:InitEntity", entityType.FullName); - try - { - DAL.WriteLog("初始化实体类：{0}", entityType.FullName); + if (throwOnError) throw; + } + } - var factory = entityType.AsFactory(); + /// 初始化指定实体类，执行反向工程检查，初始化数据 + /// + public static void InitEntity(Type entityType) + { + if (entityType == null) throw new ArgumentNullException(nameof(entityType)); - factory.Session.InitData(); - } - catch (Exception ex) - { - span?.SetError(ex, null); - throw; - } + using var span = DefaultTracer.Instance?.NewSpan("db:InitEntity", entityType.FullName); + try + { + DAL.WriteLog("初始化实体类：{0}", entityType.FullName); + + var factory = entityType.AsFactory(); + + factory.Session.InitData(); + } + catch (Exception ex) + { + span?.SetError(ex, null); + throw; } - #endregion } + #endregion } \ No newline at end of file diff --git a/XCode/Entity/Entity_Meta.cs b/XCode/Entity/Entity_Meta.cs index c21459e47..ebcac6f52 100644 --- a/XCode/Entity/Entity_Meta.cs +++ b/XCode/Entity/Entity_Meta.cs @@ -167,7 +167,7 @@ public static FieldItem Unique #region 分表分库 /// 分表分库策略 - public static IShardPolicy ShardPolicy { get; set; } + public static IShardPolicy? ShardPolicy { get; set; } /// 创建分库会话，using结束时自动还原 /// 连接名 @@ -221,10 +221,10 @@ public static IEnumerable AutoShard(DateTime start, DateTime end, Func private class SplitPackge : IDisposable { /// 连接名 - public String ConnName { get; set; } + public String? ConnName { get; set; } /// 表名 - public String TableName { get; set; } + public String? TableName { get; set; } public SplitPackge(String connName, String tableName) { diff --git a/XCode/Entity/Entity_Operate.cs b/XCode/Entity/Entity_Operate.cs index ef4591785..0bf645ec4 100644 --- a/XCode/Entity/Entity_Operate.cs +++ b/XCode/Entity/Entity_Operate.cs @@ -26,7 +26,7 @@ public class DefaultEntityFactory : IEntityFactory #endregion #region 属性 - private IEntity _Default; + private IEntity? _Default; /// 默认实体 public virtual IEntity Default { get => _Default ??= new TEntity(); set => _Default = value; } @@ -49,10 +49,10 @@ public class DefaultEntityFactory : IEntityFactory public virtual FieldItem Master => Meta.Master; /// 连接名。当前线程正在使用的连接名 - public virtual String ConnName { get => Meta.ConnName; set => Meta.ConnName = value; } + public virtual String ConnName { get => Meta.ConnName!; set => Meta.ConnName = value; } /// 表名。当前线程正在使用的表名 - public virtual String TableName { get => Meta.TableName; set => Meta.TableName = value; } + public virtual String TableName { get => Meta.TableName!; set => Meta.TableName = value; } #endregion #region 构造 @@ -69,7 +69,7 @@ public DefaultEntityFactory() /// 创建一个实体对象 /// 是否为了编辑而创建，如果是，可以再次做一些相关的初始化工作 /// - public virtual IEntity Create(Boolean forEdit = false) => (Default as TEntity).CreateInstance(forEdit); + public virtual IEntity Create(Boolean forEdit = false) => (Default as TEntity)!.CreateInstance(forEdit); /// 加载记录集 /// 记录集 @@ -82,22 +82,22 @@ public DefaultEntityFactory() /// 名称 /// 数值 /// - public virtual IEntity Find(String name, Object value) => Entity.Find(name, value); + public virtual IEntity? Find(String name, Object value) => Entity.Find(name, value); /// 根据条件查找单个实体 /// /// - public virtual IEntity Find(Expression where) => Entity.Find(where); + public virtual IEntity? Find(Expression where) => Entity.Find(where); /// 根据主键查找单个实体 /// /// - public virtual IEntity FindByKey(Object key) => Entity.FindByKey(key); + public virtual IEntity? FindByKey(Object key) => Entity.FindByKey(key); /// 根据主键查询一个实体对象用于表单编辑 /// /// - public virtual IEntity FindByKeyForEdit(Object key) => Entity.FindByKeyForEdit(key); + public virtual IEntity? FindByKeyForEdit(Object key) => Entity.FindByKeyForEdit(key); #endregion #region 静态查询 @@ -162,7 +162,7 @@ public DefaultEntityFactory() public virtual IEntitySession GetSession(String connName, String tableName) => EntitySession.Create(connName, tableName); /// 分表分库策略 - public virtual IShardPolicy ShardPolicy { get => Meta.ShardPolicy; set => Meta.ShardPolicy = value; } + public virtual IShardPolicy? ShardPolicy { get => Meta.ShardPolicy; set => Meta.ShardPolicy = value; } /// 创建分库会话，using结束时自动还原 /// 连接名 @@ -173,12 +173,12 @@ public DefaultEntityFactory() /// 针对实体对象自动分库分表 /// /// - public virtual IDisposable CreateShard(IEntity entity) => Meta.CreateShard(entity as TEntity); + public virtual IDisposable? CreateShard(IEntity entity) => Meta.CreateShard((entity as TEntity)!); /// 为实体对象、时间、雪花Id等计算分表分库 /// /// - public virtual IDisposable CreateShard(Object value) => Meta.CreateShard(value); + public virtual IDisposable? CreateShard(Object value) => Meta.CreateShard(value); /// 针对时间区间自动分库分表，常用于多表顺序查询，支持倒序 /// @@ -195,7 +195,7 @@ public DefaultEntityFactory() /// 查找函数 /// 创建对象 /// - public virtual IEntity GetOrAdd(TKey key, Func find, Func create) => Entity.GetOrAdd(key, (k, b) => find?.Invoke(k, b) as TEntity, k => create?.Invoke(k) as TEntity); + public virtual IEntity GetOrAdd(TKey key, Func find, Func create) => Entity.GetOrAdd(key, (k, b) => find?.Invoke(k, b) as TEntity, k => (create?.Invoke(k) as TEntity)!); #endregion #region 一些设置 @@ -211,15 +211,15 @@ public DefaultEntityFactory() public virtual Boolean AllowInsertIdentity { get => _AllowInsertIdentity.Value; set => _AllowInsertIdentity.Value = value; } /// 自动设置Guid的字段。对实体类有效，可在实体类类型构造函数里面设置 - public virtual FieldItem AutoSetGuidField { get; set; } + public virtual FieldItem? AutoSetGuidField { get; set; } /// 默认累加字段 public virtual ICollection AdditionalFields { get; } = new HashSet(StringComparer.OrdinalIgnoreCase); private Boolean _MasterTime_; - private FieldItem _MasterTime; + private FieldItem? _MasterTime; /// 主时间字段。代表当前数据行更新时间 - public FieldItem MasterTime + public FieldItem? MasterTime { get { @@ -234,7 +234,7 @@ public FieldItem MasterTime set { _MasterTime = value; _MasterTime_ = false; } } - private static FieldItem GetMasterTime() + private static FieldItem? GetMasterTime() { var fis = Meta.Fields.Where(e => e.Type == typeof(DateTime)).ToArray(); if (fis.Length == 0) return null; @@ -260,11 +260,11 @@ private static FieldItem GetMasterTime() } /// 默认选择的字段 - public String Selects { get; set; } + public String? Selects { get; set; } - private String _SelectStat; + private String? _SelectStat; /// 默认选择统计语句 - public String SelectStat + public String? SelectStat { get { @@ -326,7 +326,7 @@ public String SelectStat /// 雪花Id生成器。Int64主键非自增时，自动填充 public Snowflake Snow { get; } = new Snowflake(); - private SqlTemplate _Template; + private SqlTemplate? _Template; /// Sql模版 public SqlTemplate Template { diff --git a/XCode/Entity/IEntityOperate.cs b/XCode/Entity/IEntityOperate.cs index a7a40e748..4b9317cc0 100644 --- a/XCode/Entity/IEntityOperate.cs +++ b/XCode/Entity/IEntityOperate.cs @@ -81,22 +81,22 @@ public interface IEntityFactory /// 名称 /// 数值 /// - IEntity Find(String name, Object value); + IEntity? Find(String name, Object value); /// 根据条件查找单个实体 /// /// - IEntity Find(Expression where); + IEntity? Find(Expression where); /// 根据主键查找单个实体 /// /// - IEntity FindByKey(Object key); + IEntity? FindByKey(Object key); /// 根据主键查询一个实体对象用于表单编辑 /// /// - IEntity FindByKeyForEdit(Object key); + IEntity? FindByKeyForEdit(Object key); #endregion #region 静态查询 @@ -163,7 +163,7 @@ public interface IEntityFactory IEntitySession GetSession(String connName, String tableName); /// 分表分库策略 - IShardPolicy ShardPolicy { get; set; } + IShardPolicy? ShardPolicy { get; set; } /// 创建分库会话，using结束时自动还原 /// 连接名 @@ -174,12 +174,12 @@ public interface IEntityFactory /// 针对实体对象自动分库分表 /// /// - IDisposable CreateShard(IEntity entity); + IDisposable? CreateShard(IEntity entity); /// 为实体对象、时间、雪花Id等计算分表分库 /// /// - IDisposable CreateShard(Object value); + IDisposable? CreateShard(Object value); /// 针对时间区间自动分库分表，常用于多表顺序查询，支持倒序 /// @@ -207,19 +207,19 @@ public interface IEntityFactory Boolean AllowInsertIdentity { get; set; } /// 自动设置Guid的字段。对实体类有效，可在实体类类型构造函数里面设置 - FieldItem AutoSetGuidField { get; set; } + FieldItem? AutoSetGuidField { get; set; } /// 默认累加字段 ICollection AdditionalFields { get; } /// 主时间字段。代表当前数据行更新时间 - FieldItem MasterTime { get; set; } + FieldItem? MasterTime { get; set; } /// 默认选择的字段 - String Selects { get; set; } + String? Selects { get; set; } /// 默认选择统计语句 - String SelectStat { get; set; } + String? SelectStat { get; set; } /// 实体模块集合 EntityModules Modules { get; } diff --git a/XUnitTest.XCode/Membership/AreaTests.cs b/XUnitTest.XCode/Membership/AreaTests.cs index 1bf9a6174..305fb6a8c 100644 --- a/XUnitTest.XCode/Membership/AreaTests.cs +++ b/XUnitTest.XCode/Membership/AreaTests.cs @@ -157,7 +157,7 @@ public void VirtualTest() public async void Download() { //var url = "http://www.mca.gov.cn/article/sj/xzqh/2020/2020/2020092500801.html"; - var url = "https://x.newlifex.com/202201xzqh.htm"; + var url = "http://x.newlifex.com/202201xzqh.htm"; var file = "area.html".GetFullPath(); //if (!File.Exists(file)) { From 47341b6e637ac9f878b8379b381563d75423f4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=A7=E7=9F=B3=E5=A4=B4?= Date: Sun, 3 Mar 2024 13:23:51 +0800 Subject: [PATCH 6/6] =?UTF-8?q?v11.11=20=E9=87=8D=E6=9E=84=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E6=95=B0=E6=8D=AE=E6=8A=BD=E5=8F=96=E5=99=A8=EF=BC=9B?= =?UTF-8?q?=E5=A2=9E=E5=BC=BASqlite=E5=92=8CSqlServer=E7=9A=84=E6=AD=A3?= =?UTF-8?q?=E5=8F=8D=E5=90=91=E5=B7=A5=E7=A8=8B=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Test/Test.csproj | 2 +- XCode/XCode.csproj | 6 +++--- XUnitTest.XCode/XUnitTest.XCode.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Test/Test.csproj b/Test/Test.csproj index 89a9e18d2..ba0343c2a 100644 --- a/Test/Test.csproj +++ b/Test/Test.csproj @@ -40,7 +40,7 @@ - + diff --git a/XCode/XCode.csproj b/XCode/XCode.csproj index 493c1810f..bfb17ac18 100644 --- a/XCode/XCode.csproj +++ b/XCode/XCode.csproj @@ -7,7 +7,7 @@ 数据中间件，支持MySQL、SQLite、SqlServer、Oracle、Postgresql、TDengine、达梦，重点在缓存、性能、分表、自动建表。 新生命开发团队 ©2002-2024 NewLife - 11.10 + 11.11 $([System.DateTime]::Now.ToString(`yyyy.MMdd`)) $(VersionPrefix).$(VersionSuffix) $(Version) @@ -29,7 +29,7 @@ https://github.com/NewLifeX/X git 新生命团队;X组件;NewLife;$(AssemblyName) - 正反向工程的字段类型匹配更宽松；新增DataMethod重构实体过滤器；实体查询默认不再自动增加主键降序 + 重构时间数据抽取器；增强Sqlite和SqlServer的正反向工程支持 MIT true true @@ -44,7 +44,7 @@ - + diff --git a/XUnitTest.XCode/XUnitTest.XCode.csproj b/XUnitTest.XCode/XUnitTest.XCode.csproj index 882ac9649..cc7825215 100644 --- a/XUnitTest.XCode/XUnitTest.XCode.csproj +++ b/XUnitTest.XCode/XUnitTest.XCode.csproj @@ -116,7 +116,7 @@ - +