Skip to content
Permalink
Browse files

抽离MySql作为单独项目

  • Loading branch information...
Varorbc committed Jul 31, 2019
1 parent f4e72a9 commit 604267a730029cf2b974691bd3c684090f27f6d7
@@ -1,8 +1,9 @@
<Project>

<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<LangVersion>latest</LangVersion>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

</Project>
</Project>
@@ -56,6 +56,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docker", "docker", "{0096D2
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.Spiders.Startup", "src\DotnetSpider.Spiders.Startup\DotnetSpider.Spiders.Startup.csproj", "{8D23B30D-D4A7-46D1-BB33-D799B4AE96DC}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotnetSpider.MySql", "src\DotnetSpider.MySql\DotnetSpider.MySql.csproj", "{3850DAC4-F700-4E56-B415-BF6890C832CB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -110,6 +112,10 @@ Global
{8D23B30D-D4A7-46D1-BB33-D799B4AE96DC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8D23B30D-D4A7-46D1-BB33-D799B4AE96DC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8D23B30D-D4A7-46D1-BB33-D799B4AE96DC}.Release|Any CPU.Build.0 = Release|Any CPU
{3850DAC4-F700-4E56-B415-BF6890C832CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3850DAC4-F700-4E56-B415-BF6890C832CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3850DAC4-F700-4E56-B415-BF6890C832CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3850DAC4-F700-4E56-B415-BF6890C832CB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -10,7 +10,7 @@

<ItemGroup>
<ProjectReference Include="..\DotnetSpider.Kafka\DotnetSpider.Kafka.csproj" />
<ProjectReference Include="..\DotnetSpider\DotnetSpider.csproj" />
<ProjectReference Include="..\DotnetSpider.MySql\DotnetSpider.MySql.csproj" />
</ItemGroup>

<ItemGroup>
@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">

<Import Project="../../package.props" />

<PropertyGroup>
<PackageId>DotnetSpider.MySql</PackageId>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="MySqlConnector" Version="0.56.0" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\DotnetSpider\DotnetSpider.csproj" />
</ItemGroup>

</Project>
@@ -68,7 +68,7 @@ public async Task HeartbeatAsync(DownloaderAgentHeartbeat agent)
agent);
await conn.ExecuteAsync(
"UPDATE dotnetspider.downloader_agent SET last_modification_time = @LastModificationTime WHERE id = @AgentId",
new {agent.AgentId, LastModificationTime = agent.CreationTime});
new { agent.AgentId, LastModificationTime = agent.CreationTime });
}
}
}
@@ -6,7 +6,7 @@
using Microsoft.Extensions.Logging;
using MySql.Data.MySqlClient;

namespace DotnetSpider.DataFlow.Storage
namespace DotnetSpider.DataFlow.Storage.MySql
{
/// <summary>
/// MySql 保存解析(实体)结果
@@ -269,55 +269,55 @@ protected virtual string GenerateDataTypeSql(string type, int length)
switch (type)
{
case BoolType:
{
dataType = "BOOL";
break;
}
{
dataType = "BOOL";
break;
}
case DateTimeType:
case DateTimeOffsetType:
{
dataType = "TIMESTAMP DEFAULT CURRENT_TIMESTAMP";
break;
}
{
dataType = "TIMESTAMP DEFAULT CURRENT_TIMESTAMP";
break;
}

case DecimalType:
{
dataType = "DECIMAL(18,2)";
break;
}
{
dataType = "DECIMAL(18,2)";
break;
}
case DoubleType:
{
dataType = "DOUBLE";
break;
}
{
dataType = "DOUBLE";
break;
}
case FloatType:
{
dataType = "FLOAT";
break;
}
{
dataType = "FLOAT";
break;
}
case IntType:
{
dataType = "INT";
break;
}
{
dataType = "INT";
break;
}
case LongType:
{
dataType = "BIGINT";
break;
}
{
dataType = "BIGINT";
break;
}
case ByteType:
{
dataType = "INT";
break;
}
{
dataType = "INT";
break;
}
default:
{
dataType = length <= 0 || length > 8000 ? "LONGTEXT" : $"VARCHAR({length})";
break;
}
{
dataType = length <= 0 || length > 8000 ? "LONGTEXT" : $"VARCHAR({length})";
break;
}
}

return dataType;
}
}
}
}
@@ -8,7 +8,7 @@
using DotnetSpider.DataFlow.Storage.Model;
using MySql.Data.MySqlClient;

namespace DotnetSpider.DataFlow.Storage
namespace DotnetSpider.DataFlow.Storage.MySql
{
/// <summary>
/// 文件类型
@@ -49,7 +49,7 @@ public static MySqlFileEntityStorage CreateFromOptions(SpiderOptions options)
{
var fileType = string.IsNullOrWhiteSpace(options.MySqlFileType)
? MySqlFileType.InsertSql
: (MySqlFileType) Enum.Parse(typeof(MySqlFileType),
: (MySqlFileType)Enum.Parse(typeof(MySqlFileType),
options.MySqlFileType);
return new MySqlFileEntityStorage(fileType)
{
@@ -70,19 +70,19 @@ protected override Task<DataFlowResult> Store(DataFlowContext context)
{
foreach (var item in context.GetParseData())
{
var tableMetadata = (TableMetadata) context[item.Key];
var tableMetadata = (TableMetadata)context[item.Key];
switch (MySqlFileType)
{
case MySqlFileType.LoadFile:
{
WriteLoadFile(context, tableMetadata, item.Value);
break;
}
{
WriteLoadFile(context, tableMetadata, item.Value);
break;
}
case MySqlFileType.InsertSql:
{
WriteInsertFile(context, tableMetadata, item.Value);
break;
}
{
WriteInsertFile(context, tableMetadata, item.Value);
break;
}
}
}

@@ -175,4 +175,4 @@ private string GetNameSql(string name)
return IgnoreCase ? name.ToLowerInvariant() : name;
}
}
}
}
@@ -0,0 +1,26 @@
using DotnetSpider.DataFlow;
using DotnetSpider.DownloadAgentRegisterCenter;
using DotnetSpider.DownloadAgentRegisterCenter.Internal;
using DotnetSpider.Statistics;
using DotnetSpider.Statistics.Store;
using Microsoft.Extensions.DependencyInjection;

namespace DotnetSpider
{
public static class ServiceCollectionExtensions
{
public static DownloadAgentRegisterCenterBuilder UseMySqlDownloaderAgentStore(
this DownloadAgentRegisterCenterBuilder builder)
{
builder.Services.AddSingleton<IDownloaderAgentStore, MySqlDownloaderAgentStore>();
return builder;
}

public static StatisticsBuilder UseMySql(this StatisticsBuilder builder)
{
Check.NotNull(builder, nameof(builder));
builder.Services.AddSingleton<IStatisticsStore, MySqlStatisticsStore>();
return builder;
}
}
}
@@ -21,7 +21,7 @@

<ItemGroup>
<ProjectReference Include="..\DotnetSpider.Kafka\DotnetSpider.Kafka.csproj" />
<ProjectReference Include="..\DotnetSpider\DotnetSpider.csproj" />
<ProjectReference Include="..\DotnetSpider.MySql\DotnetSpider.MySql.csproj" />
</ItemGroup>


@@ -7,7 +7,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Npgsql" Version="4.0.7" />
<PackageReference Include="Npgsql" Version="4.0.7" />
</ItemGroup>

<ItemGroup>

0 comments on commit 604267a

Please sign in to comment.
You can’t perform that action at this time.