修复请求错误没有真正重试的错误

dotnetcore · Jul 11, 2023 · 3417e01 · 3417e01
1 parent 7f678d5
commit 3417e01
Show file tree

Hide file tree

Showing 12 changed files with 24 additions and 24 deletions.
diff --git a/package.props b/package.props
@@ -4,9 +4,9 @@
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
     <PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
-    <Version>5.1.1</Version>
-    <FileVersion>5.1.1</FileVersion>
-    <AssemblyVersion>5.1.1</AssemblyVersion>
+    <Version>5.1.2</Version>
+    <FileVersion>5.1.2</FileVersion>
+    <AssemblyVersion>5.1.2</AssemblyVersion>
     <Authors>zlzforever@163.com;</Authors>
     <Copyright>Copyright 2023 Lewis Zou</Copyright>
     <Description>DotnetSpider, a .NET Standard web crawling library. It is lightweight, efficient and fast high-level web crawling &amp; scraping framework</Description>

diff --git a/src/DotnetSpider.Agent/DotnetSpider.Agent.csproj b/src/DotnetSpider.Agent/DotnetSpider.Agent.csproj
@@ -20,7 +20,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
+    <PackageReference Include="Serilog.AspNetCore" Version="7.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
     <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="3.1.0" />

diff --git a/src/DotnetSpider.AgentCenter/DotnetSpider.AgentCenter.csproj b/src/DotnetSpider.AgentCenter/DotnetSpider.AgentCenter.csproj
@@ -8,8 +8,8 @@
   <Import Project="../../package.props" />
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.0" />
-    <PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Serilog.AspNetCore" Version="7.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
     <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="3.1.0" />

diff --git a/src/DotnetSpider.Mongo/DotnetSpider.Mongo.csproj b/src/DotnetSpider.Mongo/DotnetSpider.Mongo.csproj
@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="MongoDB.Driver" Version="2.19.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/DotnetSpider.MySql/DotnetSpider.MySql.csproj b/src/DotnetSpider.MySql/DotnetSpider.MySql.csproj
@@ -11,6 +11,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="MySqlConnector" Version="2.2.5" />
+    <PackageReference Include="MySqlConnector" Version="2.2.6" />
   </ItemGroup>
 </Project>
diff --git a/src/DotnetSpider.PostgreSql/DotnetSpider.PostgreSql.csproj b/src/DotnetSpider.PostgreSql/DotnetSpider.PostgreSql.csproj
@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Npgsql" Version="7.0.1" />
+    <PackageReference Include="Npgsql" Version="7.0.4" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/DotnetSpider.RabbitMQ/DotnetSpider.RabbitMQ.csproj b/src/DotnetSpider.RabbitMQ/DotnetSpider.RabbitMQ.csproj
@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="RabbitMQ.Client" Version="6.4.0" />
-    <PackageReference Include="Polly" Version="7.2.3" />
+    <PackageReference Include="RabbitMQ.Client" Version="6.5.0" />
+    <PackageReference Include="Polly" Version="7.2.4" />
   </ItemGroup>
 </Project>
diff --git a/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj b/src/DotnetSpider.Sample/DotnetSpider.Sample.csproj
@@ -14,7 +14,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
+    <PackageReference Include="Serilog.AspNetCore" Version="7.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
     <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="3.1.0" />

diff --git a/src/DotnetSpider.Spiders/DotnetSpider.Spiders.csproj b/src/DotnetSpider.Spiders/DotnetSpider.Spiders.csproj
@@ -12,7 +12,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
+    <PackageReference Include="Serilog.AspNetCore" Version="7.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
     <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
     <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="3.1.0" />

diff --git a/src/DotnetSpider.Tests/DotnetSpider.Tests.csproj b/src/DotnetSpider.Tests/DotnetSpider.Tests.csproj
@@ -6,13 +6,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
-    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
+    <PackageReference Include="xunit" Version="2.5.0" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="3.2.0">
+    <PackageReference Include="coverlet.collector" Version="6.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

diff --git a/src/DotnetSpider/DotnetSpider.csproj b/src/DotnetSpider/DotnetSpider.csproj
@@ -10,18 +10,18 @@
 
   <ItemGroup>
     <PackageReference Include="Bert.RateLimiters" Version="1.0.15" />
-    <PackageReference Include="MessagePack" Version="2.4.59" />
+    <PackageReference Include="MessagePack" Version="2.5.124" />
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
     <PackageReference Include="Microsoft.VisualBasic" Version="10.3.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
-    <PackageReference Include="Dapper" Version="2.0.123" />
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Dapper" Version="2.0.143" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.49" />
     <PackageReference Include="System.Collections.Immutable" Version="7.0.0" />
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
-    <PackageReference Include="System.Text.Json" Version="7.0.1" />
+    <PackageReference Include="System.Text.Json" Version="7.0.3" />
     <PackageReference Include="System.Threading.Channels" Version="7.0.0" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="7.0.0" />
     <PackageReference Include="ZCJ.HashedWheelTimer" Version="0.10.2" />

diff --git a/src/DotnetSpider/Spider.cs b/src/DotnetSpider/Spider.cs
@@ -296,7 +296,7 @@ private async Task RegisterConsumerAsync(CancellationToken stoppingToken)
 									Logger.LogError(
 										$"{SpiderId} download {request.RequestUri}, {request.Hash} status code: {response.StatusCode} failed: {response.ReasonPhrase}");
 
-									// 每次调用添加会导致 Requested + 1, 因此失败多次的请求最终会被过滤不再加到调度队列
+									request.RequestedTimes += 1;
 									await AddRequestsAsync(request);
 
 									OnRequestError?.Invoke(request, response);
@@ -394,7 +394,7 @@ private async Task RunAsync(CancellationToken stoppingToken)
 						sleepTime += 10;
 
 						if (await WaitForContinueAsync(sleepTime, sleepTimeLimit, (end - start).TotalSeconds,
-							$"{SpiderId} too much requests enqueued"))
+							    $"{SpiderId} too much requests enqueued"))
 						{
 							continue;
 						}