Merge branch 'master' into UseDefaultCredentials

sjdirect · Sep 12, 2018 · 195b959 · 195b959
2 parents e47539c + 080f9e3
commit 195b959
Show file tree

Hide file tree

Showing 7 changed files with 89 additions and 4 deletions.
diff --git a/Abot.Tests.Unit/App.config b/Abot.Tests.Unit/App.config
@@ -62,9 +62,48 @@
   </log4net>
 
   <abot>
-    <crawlBehavior maxConcurrentThreads="11" maxPagesToCrawl="33" maxPagesToCrawlPerDomain="333" maxPageSizeInBytes="4444" userAgentString="aaaa" crawlTimeoutSeconds="44" downloadableContentTypes="bbbb" isUriRecrawlingEnabled="true" isExternalPageCrawlingEnabled="true" isExternalPageLinksCrawlingEnabled="true" httpServicePointConnectionLimit="21" httpRequestTimeoutInSeconds="22" httpRequestMaxAutoRedirects="23" isHttpRequestAutoRedirectsEnabled="true" isHttpRequestAutomaticDecompressionEnabled="true" isSendingCookiesEnabled="true" isSslCertificateValidationEnabled="false" isRespectUrlNamedAnchorOrHashbangEnabled="true" minAvailableMemoryRequiredInMb="25" maxMemoryUsageInMb="26" maxMemoryUsageCacheTimeInSeconds="27" maxCrawlDepth="28" maxLinksPerPage="29" isForcedLinkParsingEnabled="true" maxRetryCount="4" minRetryDelayInMilliseconds="4444" />
-    <politeness isRespectRobotsDotTextEnabled="true" isRespectMetaRobotsNoFollowEnabled="true" isRespectHttpXRobotsTagHeaderNoFollowEnabled="true" isRespectAnchorRelNoFollowEnabled="true" isIgnoreRobotsDotTextIfRootDisallowedEnabled="true" robotsDotTextUserAgentString="zzzz" maxRobotsDotTextCrawlDelayInSeconds="5" minCrawlDelayPerDomainMilliSeconds="55" />
-    <authorization isAlwaysLogin="true" loginUser="test" loginPassword="dummyPass" useDefaultCredentials="false" />
+    <crawlBehavior 
+      maxConcurrentThreads="11" 
+      maxPagesToCrawl="33" 
+      maxPagesToCrawlPerDomain="333" 
+      maxPageSizeInBytes="4444" 
+      userAgentString="aaaa"
+      httpProtocolVersion="1.0"
+      crawlTimeoutSeconds="44" 
+      downloadableContentTypes="bbbb" 
+      isUriRecrawlingEnabled="true" 
+      isExternalPageCrawlingEnabled="true" 
+      isExternalPageLinksCrawlingEnabled="true" 
+      httpServicePointConnectionLimit="21" 
+      httpRequestTimeoutInSeconds="22" 
+      httpRequestMaxAutoRedirects="23" 
+      isHttpRequestAutoRedirectsEnabled="true" 
+      isHttpRequestAutomaticDecompressionEnabled="true" 
+      isSendingCookiesEnabled="true" 
+      isSslCertificateValidationEnabled="false" 
+      isRespectUrlNamedAnchorOrHashbangEnabled="true" 
+      minAvailableMemoryRequiredInMb="25" 
+      maxMemoryUsageInMb="26" 
+      maxMemoryUsageCacheTimeInSeconds="27" 
+      maxCrawlDepth="28" 
+      maxLinksPerPage="29" 
+      isForcedLinkParsingEnabled="true" 
+      maxRetryCount="4" 
+      minRetryDelayInMilliseconds="4444" />
+    <politeness 
+      isRespectRobotsDotTextEnabled="true" 
+      isRespectMetaRobotsNoFollowEnabled="true" 
+      isRespectHttpXRobotsTagHeaderNoFollowEnabled="true" 
+      isRespectAnchorRelNoFollowEnabled="true" 
+      isIgnoreRobotsDotTextIfRootDisallowedEnabled="true" 
+      robotsDotTextUserAgentString="zzzz" 
+      maxRobotsDotTextCrawlDelayInSeconds="5" 
+      minCrawlDelayPerDomainMilliSeconds="55" />
+    <authorization 
+      isAlwaysLogin="true" 
+      loginUser="test" 
+      loginPassword="dummyPass"
+      useDefaultCredentials="false" />
     <extensionValues>
       <add key="key1" value="value1" />
       <add key="key2" value="value2" />

diff --git a/Abot.Tests.Unit/Core/AbotConfigurationSectionHandlerTest.cs b/Abot.Tests.Unit/Core/AbotConfigurationSectionHandlerTest.cs
@@ -21,6 +21,7 @@ public void GetSetion_FillsConfigValuesFromAppConfigFile()
             Assert.AreEqual(333, _uut.CrawlBehavior.MaxPagesToCrawlPerDomain);
             Assert.AreEqual(4444, _uut.CrawlBehavior.MaxPageSizeInBytes);
             Assert.AreEqual("aaaa", _uut.CrawlBehavior.UserAgentString);
+            Assert.AreEqual("1.0", _uut.CrawlBehavior.HttpProtocolVersion);
             Assert.AreEqual(true, _uut.CrawlBehavior.IsExternalPageCrawlingEnabled);
             Assert.AreEqual(true, _uut.CrawlBehavior.IsExternalPageLinksCrawlingEnabled);
             Assert.AreEqual(21, _uut.CrawlBehavior.HttpServicePointConnectionLimit);
@@ -71,6 +72,7 @@ public void Convert_CovertsFromSectionObjectToDtoObject()
             Assert.AreEqual(result.MaxPagesToCrawlPerDomain, _uut.CrawlBehavior.MaxPagesToCrawlPerDomain);
             Assert.AreEqual(result.MaxPageSizeInBytes, _uut.CrawlBehavior.MaxPageSizeInBytes);
             Assert.AreEqual(result.UserAgentString, _uut.CrawlBehavior.UserAgentString);
+            Assert.AreEqual(result.HttpProtocolVersion, HttpProtocolVersion.Version10);
             Assert.AreEqual(result.IsExternalPageCrawlingEnabled, _uut.CrawlBehavior.IsExternalPageCrawlingEnabled);
             Assert.AreEqual(result.IsExternalPageLinksCrawlingEnabled, _uut.CrawlBehavior.IsExternalPageLinksCrawlingEnabled);
             Assert.AreEqual(result.HttpServicePointConnectionLimit, _uut.CrawlBehavior.HttpServicePointConnectionLimit);

diff --git a/Abot/Abot.csproj b/Abot/Abot.csproj
@@ -96,6 +96,7 @@
     <Compile Include="Core\CompactCrawledUrlRepository.cs" />
     <Compile Include="Core\AngleSharpHyperLinkParser.cs" />
     <Compile Include="Crawler\RobotsDotTextParseCompletedArgs.cs" />
+    <Compile Include="Poco\Enums.cs" />
     <Compile Include="Poco\HttpWebResponseWrapper.cs" />
     <Compile Include="Util\BloomFilter.cs" />
     <Compile Include="Util\CachedMemoryMonitor.cs" />

diff --git a/Abot/Core/AbotConfigurationSectionHandler.cs b/Abot/Core/AbotConfigurationSectionHandler.cs
@@ -58,6 +58,7 @@ private void Map(CrawlBehaviorElement src, CrawlConfiguration dest)
             dest.MaxPagesToCrawlPerDomain = src.MaxPagesToCrawlPerDomain;
             dest.MaxPageSizeInBytes = src.MaxPageSizeInBytes;
             dest.UserAgentString = src.UserAgentString;
+            dest.HttpProtocolVersion = GetHttpProtocolVersion(src);
             dest.CrawlTimeoutSeconds = src.CrawlTimeoutSeconds;
             dest.IsUriRecrawlingEnabled = src.IsUriRecrawlingEnabled;
             dest.IsExternalPageCrawlingEnabled = src.IsExternalPageCrawlingEnabled;
@@ -101,6 +102,20 @@ private void Map(AuthorizationElement src, CrawlConfiguration dest)
             dest.UseDefaultCredentials = src.UseDefaultCredentials;
         }
 
+        private HttpProtocolVersion GetHttpProtocolVersion(CrawlBehaviorElement src)
+        {
+
+            switch (src.HttpProtocolVersion)
+            {
+                case "1.0":
+                    return HttpProtocolVersion.Version10;
+                case "1.1":
+                    return HttpProtocolVersion.Version11;
+                default:
+                    return HttpProtocolVersion.NotSpecified;
+            }
+        }
+
         public static AbotConfigurationSectionHandler LoadFromXml()
         {
             return ((AbotConfigurationSectionHandler)System.Configuration.ConfigurationManager.GetSection("abot"));
@@ -231,6 +246,12 @@ public string UserAgentString
             get { return (string)this["userAgentString"]; }
         }
 
+        [ConfigurationProperty("httpProtocolVersion", IsRequired = false)]
+        public string HttpProtocolVersion
+        {
+            get{ return (string)this["httpProtocolVersion"]; }
+        }
+
         [ConfigurationProperty("crawlTimeoutSeconds", IsRequired = false)]
         public int CrawlTimeoutSeconds
         {

diff --git a/Abot/Core/PageRequester.cs b/Abot/Core/PageRequester.cs
@@ -204,7 +204,7 @@ protected virtual HttpWebRequest BuildRequestObject(Uri uri)
             request.AllowAutoRedirect = _config.IsHttpRequestAutoRedirectsEnabled;
             request.UserAgent = _config.UserAgentString;
             request.Accept = "*/*";
-            request.ProtocolVersion = HttpVersion.Version10; //https://github.com/sjdirect/abot/issues/187
+            request.ProtocolVersion = GetEquivalentHttpProtocolVersion();
 
             if (_config.HttpRequestMaxAutoRedirects > 0)
                 request.MaximumAutomaticRedirections = _config.HttpRequestMaxAutoRedirects;
@@ -238,6 +238,14 @@ protected virtual HttpWebRequest BuildRequestObject(Uri uri)
             return request;
         }
 
+        private Version GetEquivalentHttpProtocolVersion()
+        {
+            if (_config.HttpProtocolVersion == Abot.Poco.HttpProtocolVersion.Version10)
+                return HttpVersion.Version10;
+
+            return HttpVersion.Version11;
+        }
+
         protected virtual void ProcessResponseObject(HttpWebResponse response)
         {
             if (response != null && _config.IsSendingCookiesEnabled)

diff --git a/Abot/Poco/CrawlConfiguration.cs b/Abot/Poco/CrawlConfiguration.cs
@@ -53,6 +53,11 @@ public CrawlConfiguration()
         /// </summary>
         public string UserAgentString { get; set; }
 
+        /// <summary>
+        /// The http protocol version number to use during http requests. Currently supporting values "1.1" and "1.0". 
+        /// </summary>
+        public HttpProtocolVersion HttpProtocolVersion { get; set; }
+
         /// <summary>
         /// Maximum seconds before the crawl times out and stops. 
         /// If zero, this setting has no effect.

diff --git a/Abot/Poco/Enums.cs b/Abot/Poco/Enums.cs
@@ -0,0 +1,9 @@
+namespace Abot.Poco
+{
+    public enum HttpProtocolVersion
+    {
+        NotSpecified,
+        Version10,
+        Version11,
+    }
+}