core: refactor http webclient part 13 #8529 (#8941)

Merge string/byte request code
Jackett · Sep 19, 2020 · 4d63fa8 · 4d63fa8
1 parent 22bced9
commit 4d63fa8
Show file tree

Hide file tree

Showing 91 changed files with 329 additions and 398 deletions.
diff --git a/src/Jackett.Common/Indexers/Abnormal.cs b/src/Jackett.Common/Indexers/Abnormal.cs
@@ -21,6 +21,7 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
+using WebRequest = Jackett.Common.Utils.Clients.WebRequest;
 
 namespace Jackett.Common.Indexers
 {
@@ -166,7 +167,7 @@ public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken
             // Perform loggin
             latencyNow();
             output("\nPerform loggin.. with " + LoginUrl);
-            var response = await webclient.GetString(request);
+            var response = await webclient.GetResultAsync(request);
 
             // Test if we are logged in
             await ConfigureIfOK(response.Cookies, response.Cookies.Contains("session="), () =>
@@ -548,14 +549,12 @@ private async Task<string> queryCache(string request)
         /// <returns>Results from query</returns>
         private async Task<string> queryTracker(string request)
         {
-            WebResult results = null;
-
             // Cache mode not enabled or cached file didn't exist for our query
             output("\nQuerying tracker for results....");
 
             // Request our first page
             latencyNow();
-            results = await RequestStringWithCookiesAndRetry(request, null, null, emulatedBrowserHeaders);
+            var results = await RequestWithCookiesAndRetryAsync(request, headers: emulatedBrowserHeaders);
 
             // Return results from tracker
             return results.ContentString;

diff --git a/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs b/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs
@@ -10,6 +10,7 @@
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 using WebClient = Jackett.Common.Utils.Clients.WebClient;
@@ -148,7 +149,7 @@ private async Task RenewalTokenAsync()
                 { "password", configData.Password.Value.Trim() },
                 { "pid", configData.Pid.Value.Trim() }
             };
-            var result = await PostDataWithCookies(AuthUrl, body, headers: AuthHeaders);
+            var result = await WebRequestWithCookiesAsync(AuthUrl, method: RequestType.POST, data: body, headers: AuthHeaders);
             var json = JObject.Parse(result.ContentString);
             _token = json.Value<string>("token");
             if (_token == null)
@@ -161,11 +162,11 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
             var qc = GetSearchQueryParameters(query);
             var episodeSearchUrl = SearchUrl + "?" + qc.GetQueryString();
-            var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders());
+            var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders());
             if (response.Status == HttpStatusCode.Unauthorized || response.Status == HttpStatusCode.PreconditionFailed)
             {
                 await RenewalTokenAsync();
-                response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders());
+                response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders());
             }
             else if (response.Status != HttpStatusCode.OK)
                 throw new Exception($"Unknown error: {response.ContentString}");

diff --git a/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs b/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs
@@ -75,7 +75,7 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
             searchUrl += "?" + queryCollection.GetQueryString();
 
-            var response = await RequestStringWithCookiesAndRetry(searchUrl);
+            var response = await RequestWithCookiesAndRetryAsync(searchUrl);
 
             JObject json = null;
             try

diff --git a/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs b/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs
@@ -12,6 +12,7 @@
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 using WebClient = Jackett.Common.Utils.Clients.WebClient;
@@ -196,12 +197,12 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
             searchUrl += "?" + queryCollection.GetQueryString();
 
-            var response = await RequestStringWithCookiesAndRetry(searchUrl);
+            var response = await RequestWithCookiesAndRetryAsync(searchUrl);
             if (response.IsRedirect)
             {
                 // re-login
                 await ApplyConfiguration(null);
-                response = await RequestStringWithCookiesAndRetry(searchUrl);
+                response = await RequestWithCookiesAndRetryAsync(searchUrl);
             }
 
             try

diff --git a/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs b/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs
@@ -10,6 +10,7 @@
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
@@ -74,7 +75,8 @@ private async Task RenewalTokenAsync()
                 { "password", configData.Password.Value.Trim() }
             };
             var jsonData = JsonConvert.SerializeObject(body);
-            var result = await PostDataWithCookies(LoginUrl, null, headers: ApiHeaders, rawbody: jsonData);
+            var result = await WebRequestWithCookiesAsync(
+                LoginUrl, method: RequestType.POST, headers: ApiHeaders, rawbody: jsonData);
             var json = JObject.Parse(result.ContentString);
             _token = json.Value<string>("token");
             if (_token == null)
@@ -105,11 +107,11 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
                 await RenewalTokenAsync();
 
             var searchUrl = SearchUrl + "?" + qc.GetQueryString();
-            var response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders());
+            var response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
             if (response.Status == HttpStatusCode.Unauthorized)
             {
                 await RenewalTokenAsync(); // re-login
-                response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders());
+                response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
             }
             else if (response.Status != HttpStatusCode.OK)
                 throw new Exception($"Unknown error in search: {response.ContentString}");
@@ -167,11 +169,11 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
         public override async Task<byte[]> Download(Uri link)
         {
-            var response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders());
+            var response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
             if (response.Status == HttpStatusCode.Unauthorized)
             {
                 await RenewalTokenAsync();
-                response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders());
+                response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
             }
             else if (response.Status != HttpStatusCode.OK)
                 throw new Exception($"Unknown error in download: {response.ContentBytes}");

diff --git a/src/Jackett.Common/Indexers/AniDUB.cs b/src/Jackett.Common/Indexers/AniDUB.cs
@@ -145,7 +145,7 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
         private async Task EnsureAuthorized()
         {
-            var result = await RequestStringWithCookiesAndRetry(SiteLink);
+            var result = await RequestWithCookiesAndRetryAsync(SiteLink);
 
             if (!IsAuthorized(result))
             {
@@ -156,8 +156,7 @@ private async Task EnsureAuthorized()
         private async Task<List<ReleaseInfo>> FetchNewReleases()
         {
             const string ReleaseLinksSelector = "#dle-content > .story > .story_h > .lcol > h2 > a";
-
-            var result = await RequestStringWithCookiesAndRetry(SiteLink);
+            var result = await RequestWithCookiesAndRetryAsync(SiteLink);
             var releases = new List<ReleaseInfo>();
 
             try
@@ -195,7 +194,7 @@ private async Task<List<ReleaseInfo>> FetchShowReleases(string url)
                 return releases;
             }
 
-            var result = await RequestStringWithCookiesAndRetry(url);
+            var result = await RequestWithCookiesAndRetryAsync(url);
 
             try
             {
@@ -529,8 +528,7 @@ private async Task<List<ReleaseInfo>> PerformSearch(TorznabQuery query)
             const string searchLinkSelector = "#dle-content > .searchitem > h3 > a";
 
             var releases = new List<ReleaseInfo>();
-
-            var response = await PostDataWithCookiesAndRetry(SearchUrl, PreparePostData(query));
+            var response = await RequestWithCookiesAndRetryAsync(SearchUrl, method: RequestType.POST, data: PreparePostData(query));
 
             try
             {

diff --git a/src/Jackett.Common/Indexers/Anidex.cs b/src/Jackett.Common/Indexers/Anidex.cs
@@ -14,6 +14,7 @@
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
@@ -150,13 +151,13 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
             // Make search request
             var searchUri = GetAbsoluteUrl("?" + queryParameters.GetQueryString());
-            var response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri);
+            var response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri);
 
             // Check for DDOS Guard
             if (response.Status == System.Net.HttpStatusCode.Forbidden)
             {
                 await ConfigureDDoSGuardCookie();
-                response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri);
+                response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri);
             }
 
             if (response.Status != System.Net.HttpStatusCode.OK)
@@ -216,7 +217,7 @@ private IEnumerable<ReleaseInfo> ParseResult(string response)
         private async Task ConfigureDDoSGuardCookie()
         {
             const string ddosPostUrl = "https://check.ddos-guard.net/check.js";
-            var response = await RequestStringWithCookies(ddosPostUrl, string.Empty);
+            var response = await WebRequestWithCookiesAsync(ddosPostUrl, string.Empty);
             if (response.Status != System.Net.HttpStatusCode.OK)
                 throw new WebException($"Unexpected DDOS Guard response: Status: {response.Status}", WebExceptionStatus.ProtocolError);
             if (response.IsRedirect)

diff --git a/src/Jackett.Common/Indexers/AnimeBytes.cs b/src/Jackett.Common/Indexers/AnimeBytes.cs
@@ -12,6 +12,7 @@
 using Jackett.Common.Models.IndexerConfig.Bespoke;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
@@ -179,7 +180,7 @@ private async Task<IEnumerable<ReleaseInfo>> GetResults(TorznabQuery query, stri
             }
 
             // Get the content from the tracker
-            var response = await RequestStringWithCookiesAndRetry(queryUrl);
+            var response = await RequestWithCookiesAndRetryAsync(queryUrl);
             if (!response.ContentString.StartsWith("{")) // not JSON => error
                 throw new ExceptionWithConfigData("unexcepted response (not JSON)", configData);
             dynamic json = JsonConvert.DeserializeObject<dynamic>(response.ContentString);

diff --git a/src/Jackett.Common/Indexers/AnimeTorrents.cs b/src/Jackett.Common/Indexers/AnimeTorrents.cs
@@ -1,4 +1,4 @@
-using System;
+ using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Diagnostics.CodeAnalysis;
@@ -79,7 +79,7 @@ public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken
                 { "rememberme[]", "1" }
             };
 
-            var loginPage = await RequestStringWithCookiesAndRetry(LoginUrl, "", LoginUrl);
+            var loginPage = await RequestWithCookiesAndRetryAsync(LoginUrl, "", RequestType.GET, LoginUrl);
 
             var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true);
             await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("logout.php"), () =>
@@ -117,7 +117,8 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
                 { "X-Requested-With", "XMLHttpRequest" }
             };
 
-            var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders);
+            var response = await RequestWithCookiesAndRetryAsync(
+                searchUrl, referer: SearchUrlReferer, headers: extraHeaders);
 
             var results = response.ContentString;
             try

diff --git a/src/Jackett.Common/Indexers/Anthelion.cs b/src/Jackett.Common/Indexers/Anthelion.cs
@@ -94,7 +94,7 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
                 qc.Add($"filter_cat[{cat}]", "1");
 
             var searchUrl = BrowseUrl + "?" + qc.GetQueryString();
-            var results = await RequestStringWithCookies(searchUrl);
+            var results = await WebRequestWithCookiesAsync(searchUrl);
             try
             {
                 var parser = new HtmlParser();

diff --git a/src/Jackett.Common/Indexers/AwesomeHD.cs b/src/Jackett.Common/Indexers/AwesomeHD.cs
@@ -100,7 +100,7 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
             }
 
             var searchUrl = SearchUrl + "?" + qc.GetQueryString();
-            var results = await RequestStringWithCookies(searchUrl);
+            var results = await WebRequestWithCookiesAsync(searchUrl);
             if (string.IsNullOrWhiteSpace(results.ContentString))
                 throw new Exception("Empty response. Please, check the Passkey.");
 

diff --git a/src/Jackett.Common/Indexers/BB.cs b/src/Jackett.Common/Indexers/BB.cs
@@ -122,8 +122,8 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
                 request_urls.Add(SearchUrl + queryCollection.GetQueryString());
             }
-            var downloadTasksQuery =
-                from url in request_urls select RequestStringWithCookiesAndRetry(url);
+
+            var downloadTasksQuery = from url in request_urls select RequestWithCookiesAndRetryAsync(url);
 
             var responses = await Task.WhenAll(downloadTasksQuery.ToArray());
 
@@ -134,7 +134,7 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
                 if (results.IsRedirect)
                 {
                     await ApplyConfiguration(null);
-                    results = await RequestStringWithCookiesAndRetry(request_urls[i]);
+                    results = await RequestWithCookiesAndRetryAsync(request_urls[i]);
                 }
                 try
                 {

diff --git a/src/Jackett.Common/Indexers/BJShare.cs b/src/Jackett.Common/Indexers/BJShare.cs
@@ -257,12 +257,12 @@ private async Task<List<ReleaseInfo>> ParseUserSearchAsync(TorznabQuery query)
             foreach (var cat in MapTorznabCapsToTrackers(query))
                 queryCollection.Add("filter_cat[" + cat + "]", "1");
             searchUrl += "?" + queryCollection.GetQueryString();
-            var results = await RequestStringWithCookies(searchUrl);
+            var results = await WebRequestWithCookiesAsync(searchUrl);
             if (IsSessionIsClosed(results))
             {
                 // re-login
                 await ApplyConfiguration(null);
-                results = await RequestStringWithCookies(searchUrl);
+                results = await WebRequestWithCookiesAsync(searchUrl);
             }
 
             try
@@ -389,12 +389,12 @@ private async Task<List<ReleaseInfo>> ParseUserSearchAsync(TorznabQuery query)
         private async Task<List<ReleaseInfo>> ParseLast24HoursAsync()
         {
             var releases = new List<ReleaseInfo>();
-            var results = await RequestStringWithCookies(TodayUrl);
+            var results = await WebRequestWithCookiesAsync(TodayUrl);
             if (IsSessionIsClosed(results))
             {
                 // re-login
                 await ApplyConfiguration(null);
-                results = await RequestStringWithCookies(TodayUrl);
+                results = await WebRequestWithCookiesAsync(TodayUrl);
             }
 
             try

diff --git a/src/Jackett.Common/Indexers/BakaBT.cs b/src/Jackett.Common/Indexers/BakaBT.cs
@@ -14,6 +14,7 @@
 using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
+using WebRequest = Jackett.Common.Utils.Clients.WebRequest;
 
 namespace Jackett.Common.Indexers
 {
@@ -68,7 +69,7 @@ public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken
 
         private async Task DoLogin()
         {
-            var loginForm = await webclient.GetString(new Utils.Clients.WebRequest()
+            var loginForm = await webclient.GetResultAsync(new Utils.Clients.WebRequest()
             {
                 Url = LoginUrl,
                 Type = RequestType.GET
@@ -106,12 +107,12 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
             var releases = new List<ReleaseInfo>();
             var searchString = queryCopy.SanitizedSearchTerm;
             var episodeSearchUrl = SearchUrl + WebUtility.UrlEncode(searchString);
-            var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
+            var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl);
             if (!response.ContentString.Contains(LogoutStr))
             {
                 //Cookie appears to expire after a period of time or logging in to the site via browser
                 await DoLogin();
-                response = await RequestStringWithCookiesAndRetry(episodeSearchUrl);
+                response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl);
             }
 
             try
@@ -246,15 +247,15 @@ private string GetCategoryName(IElement row)
 
         public override async Task<byte[]> Download(Uri link)
         {
-            var downloadPage = await RequestStringWithCookies(link.ToString());
+            var downloadPage = await WebRequestWithCookiesAsync(link.ToString());
             var parser = new HtmlParser();
             var dom = parser.ParseDocument(downloadPage.ContentString);
             var downloadLink = dom.QuerySelectorAll(".download_link").First().GetAttribute("href");
 
             if (string.IsNullOrWhiteSpace(downloadLink))
                 throw new Exception("Unable to find download link.");
 
-            var response = await RequestBytesWithCookies(SiteLink + downloadLink);
+            var response = await WebRequestWithCookiesAsync(SiteLink + downloadLink);
             return response.ContentBytes;
         }
     }