Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

merge #1

Merged
merged 8 commits into from
Mar 4, 2024
Merged

merge #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5276d9b
时间抽取器从StartTime开始取，是大于等于
nnhy Feb 27, 2024
6242f37
[refactor] 重构时间抽取器，简化相同时间跨页逻辑，按时间分片查询。如果有多页，则分片内使用分页查询
nnhy Feb 27, 2024
ac47e81
非自增的整数主键，也可以使用Id抽取器，例如雪花Id
nnhy Feb 27, 2024
fb74b37
[fix] 修正上一次对SqlServer的修改导致DbMetaData报错的bug
nnhy Feb 28, 2024
ff8a738
Merge pull request #23 from Fat-Snail/master
nnhy Feb 28, 2024
384bdf4
合并
nnhy Feb 28, 2024
0bd94e5
确保CreateFactory返回对象唯一，否则影响Snowflake
nnhy Feb 28, 2024
47341b6
v11.11 重构时间数据抽取器；增强Sqlite和SqlServer的正反向工程支持
nnhy Mar 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Test/Test.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
<PackageReference Include="MySql.Data" Version="8.3.0" />
<PackageReference Include="NewLife.Redis" Version="5.6.2024.203" />
<PackageReference Include="NewLife.Remoting" Version="2.7.2024.202" />
<PackageReference Include="NewLife.Stardust" Version="2.9.2024.101" />
<PackageReference Include="NewLife.Stardust" Version="2.9.2024.218" />
<PackageReference Include="Oracle.ManagedDataAccess.Core" Version="3.21.130" />
<PackageReference Include="SapHana.DotNetCore.Data.Provider" Version="2.11.14" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
Expand Down
42 changes: 29 additions & 13 deletions XCode/DataAccessLayer/Database/SqlServer.cs
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,10 +764,14 @@ private List<IDataParameter[]> GetParametersList(IDataColumn[] columns, ICollect
var dt = val.ToDateTime();
if (dt.Year < 1970) val = new DateTime(1970, 1, 1);
}

if (val == null && dc.DataType == typeof(Guid))
//byte[]类型查询时候参数化异常
else if (dc.DataType == typeof(Byte[]))
{
val ??= new Byte[0];
}
if (dc.DataType == typeof(Guid))
{
val = Guid.Empty;
val ??= Guid.Empty;
}

// 逐列创建参数对象
Expand Down Expand Up @@ -991,33 +995,36 @@ public override String FieldClause(IDataTable table, Int32 index, Boolean onlyDe
// 每种数据库的自增差异太大，理应由各自处理，而不采用原始值
if (Database.Type == field.Table.DbType && !field.Identity) typeName = field.RawType;

if (String.IsNullOrEmpty(typeName)) typeName = GetFieldType(field);
if (typeName.IsNullOrEmpty()) typeName = GetFieldType(field);

sb.Append(typeName);

//增加长度
if (field.DataType.Name == "String" && !typeName.Contains("text"))
// 增加长度
if (field.DataType == typeof(String) && !typeName.IsNullOrEmpty() && !typeName.Contains("(") && !typeName.Contains("text"))
{
if (field.Length > 0)
{
sb.AppendFormat("({0})", field.Length);
}
else
{
sb.Append("(max)");
}
}
// 约束
sb.Append(GetFieldConstraints(field, onlyDefine));

return sb.ToString();
}

private DataTable AllFields = null;
private DataTable AllIndexes = null;
private DataTable? AllFields = null;
private DataTable? AllIndexes = null;

protected override void FixField(IDataColumn field, DataRow dr)
{
//修复sqlserver同步到sqlserver建表长度为1的bug
var rawType = field.RawType?.ToLower();
if (rawType == "nvarchar" || rawType == "varchar" || rawType == "char" || rawType == "binary")
field.RawType += "(" + field.Length + ")";
if (rawType == "varbinary" && field.Length == -1)
field.RawType = "varbinary(max)";

base.FixField(field, dr);

var rows = AllFields?.Select("表名='" + field.Table.TableName + "' And 字段名='" + field.ColumnName + "'", null);
Expand Down Expand Up @@ -1067,7 +1074,7 @@ public override String CreateTableSQL(IDataTable table)

public override String FieldClause(IDataColumn field, Boolean onlyDefine)
{
if (!String.IsNullOrEmpty(field.RawType) && field.RawType.Contains("char(-1)"))
if (!field.RawType.IsNullOrEmpty() && field.RawType.Contains("char(-1)"))
{
//if (IsSQL2005)
field.RawType = field.RawType.Replace("char(-1)", "char(MAX)");
Expand Down Expand Up @@ -1116,6 +1123,15 @@ public override String FieldClause(IDataColumn field, Boolean onlyDefine)
// if (field.DataType == typeof(String) && pi == "-1" && IsSQL2005) return "MAX";
// return pi;
//}

protected override String? GetFieldType(IDataColumn field)
{
//mysql 转sqlserver
if (field.RawType?.ToLower() == "longblob")
return "varbinary(max)";

return base.GetFieldType(field);
}
#endregion

#region 取得字段信息的SQL模版
Expand Down
24 changes: 17 additions & 7 deletions XCode/DataAccessLayer/DbPackage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public virtual Int32 Backup(IDataTable table, Stream stream)
{
foreach (var dt in extracer.Fetch())
{
var row = extracer.Row;
var row = extracer.TotalCount;
var count = dt.Rows.Count;
WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, connName, row, count);
if (count == 0) break;
Expand Down Expand Up @@ -161,16 +161,26 @@ protected virtual IExtracter<DbTable> GetExtracter(IDataTable table)
var id = table.Columns.FirstOrDefault(e => e.Identity);
if (id == null)
{
var pks = table.Columns.Where(e => e.PrimaryKey).ToList();
if (pks.Count == 1 && pks[0].DataType.IsInt()) id = pks[0];
var pks = table.PrimaryKeys;
if (pks.Length >= 1 && pks[0].DataType.IsInt()) id = pks[0];
}
if (id != null)
return new IdExtracter(Dal, tableName, id);

// 时间索引抽取
var time = table.Indexes.FirstOrDefault(e => table.GetColumn(e.Columns[0]).DataType == typeof(DateTime));
IDataColumn? time = null;
foreach (var dx in table.Indexes)
{
var column = table.GetColumn(dx.Columns[0]);
if (column != null && column.DataType == typeof(DateTime))
{
time = column;
break;
}
}
//var time = table.Indexes.FirstOrDefault(e => table.GetColumn(e.Columns[0])?.DataType == typeof(DateTime));
if (time != null)
return new TimeExtracter(Dal, tableName, table.GetColumn(time.Columns[0]));
return new TimeExtracter(Dal, tableName, time);

// 主键分页功能
var pk = table.Columns.FirstOrDefault(e => e.PrimaryKey);
Expand All @@ -188,7 +198,7 @@ protected virtual IExtracter<DbTable> GetExtracter(IDataTable table)

// 默认第一个字段
var dc = table.Columns.FirstOrDefault();
return dc != null ? new PagingExtracter(Dal, tableName, dc.ColumnName) : (IExtracter<DbTable>)new PagingExtracter(Dal, tableName);
return new PagingExtracter(Dal, tableName, dc?.ColumnName);
}

/// <summary>备份单表数据到文件</summary>
Expand Down Expand Up @@ -518,7 +528,7 @@ public virtual Int32 Sync(IDataTable table, String connName, Boolean syncSchema

foreach (var dt in extracer.Fetch())
{
var row = extracer.Row;
var row = extracer.TotalCount;
var count = dt.Rows.Count;
WriteLog("同步[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, Dal.ConnName, row, count);

Expand Down
24 changes: 5 additions & 19 deletions XCode/DataAccessLayer/MetaData/DbMetaData_Positive.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Data;
using System.Data.Common;
using System.Linq;
using NewLife;
using NewLife.Log;
using NewLife.Serialization;
using XCode.Common;
using XCode.Exceptions;

Expand Down Expand Up @@ -212,17 +207,9 @@ protected virtual List<IDataColumn> GetFields(IDataTable table, DataRow[] rows)

// 原始数据类型
field.RawType = GetDataRowValue<String>(dr, "DATA_TYPE", "DATATYPE", "COLUMN_DATA_TYPE");

// 长度
field.Length = GetDataRowValue<Int32>(dr, "CHARACTER_MAXIMUM_LENGTH", "LENGTH", "COLUMN_SIZE");

//修复sqlserver同步到sqlserver建表长度为1的bug
var rawType = field.RawType.ToLower();
if (rawType == "nvarchar" || rawType == "varchar" || rawType == "char" || rawType == "binary")
field.RawType += "(" + field.Length + ")";
if (rawType == "varbinary" && field.Length == -1)
field.RawType = "varbinary(max)";

if (field is XField fi)
{
// 精度 与 位数
Expand All @@ -231,7 +218,6 @@ protected virtual List<IDataColumn> GetFields(IDataTable table, DataRow[] rows)
if (field.Length == 0) field.Length = fi.Precision;
}


// 允许空
if (TryGetDataRowValue(dr, "IS_NULLABLE", out b))
field.Nullable = b;
Expand Down Expand Up @@ -344,7 +330,7 @@ protected virtual void FixIndex(IDataIndex index, DataRow dr) { }
protected IDictionary<Type, String[]> Types { get; set; } = null!;

protected List<KeyValuePair<Type, Type>>? _FieldTypeMaps;
/// <summary>字段类型映射</summary>
/// <summary>字段类型映射（数据库-实体类）</summary>
protected virtual List<KeyValuePair<Type, Type>> FieldTypeMaps
{
get
Expand All @@ -359,6 +345,8 @@ protected virtual List<KeyValuePair<Type, Type>> FieldTypeMaps
// 因为等价，字节需要能够互相映射
new(typeof(Byte), typeof(SByte)),
new(typeof(Byte), typeof(Boolean)),
new(typeof(Boolean), typeof(Byte)),
new(typeof(Byte), typeof(Int32)),

new(typeof(UInt16), typeof(Int16)),
new(typeof(Int16), typeof(UInt16)),
Expand All @@ -377,6 +365,7 @@ protected virtual List<KeyValuePair<Type, Type>> FieldTypeMaps
new(typeof(Int64), typeof(UInt64)),
//list.Add(new KeyValuePair<Type, Type>(typeof(UInt64), typeof(Int32)));
//list.Add(new KeyValuePair<Type, Type>(typeof(Int64), typeof(Int32)));
new(typeof(Int64), typeof(Int32)),

// 数据库使用Double，实体类使用Decimal
new(typeof(Double), typeof(Decimal)),
Expand Down Expand Up @@ -419,9 +408,6 @@ protected virtual List<KeyValuePair<Type, Type>> FieldTypeMaps
var typeName = ns.FirstOrDefault();
// 大文本选第二个类型
if (ns.Length > 1 && type == typeof(String) && (field.Length <= 0 || field.Length >= Database.LongTextLength)) typeName = ns[1];
//mysql 转sqlserver
if (field.RawType.ToLower() == "longblob")
typeName = "varbinary(max)";
if (typeName.Contains("{0}"))
{
if (typeName.Contains("{1}"))
Expand Down
Loading