Skip to content

Commit

Permalink
core: refactor http webclient part 9 #8529 (#7727)
Browse files Browse the repository at this point in the history 
Move identical functions up to base WebClient instance. Replay #7691
  • Loading branch information
@cadatoiva @ngosang
cadatoiva authored and ngosang committed Sep 19, 2020
1 parent 1184a92 commit c8b91be
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 419 deletions.
106 changes: 1 addition & 105 deletions src/Jackett.Common/Utils/Clients/HttpWebClient.cs
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,12 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CloudflareSolverRe;
using com.LandonKey.SocksWebProxy;
using com.LandonKey.SocksWebProxy.Proxy;
using Jackett.Common.Helpers;
using Jackett.Common.Models.Config;
using Jackett.Common.Services.Interfaces;
Expand All @@ -21,106 +16,19 @@ namespace Jackett.Common.Utils.Clients
{
public class HttpWebClient : WebClient
{
protected static Dictionary<string, ICollection<string>> trustedCertificates = new Dictionary<string, ICollection<string>>();
protected static string webProxyUrl;
protected static IWebProxy webProxy;

[DebuggerNonUserCode] // avoid "Exception User-Unhandled" Visual Studio messages
public static bool ValidateCertificate(HttpRequestMessage request, X509Certificate2 certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
{
var hash = certificate.GetCertHashString();


trustedCertificates.TryGetValue(hash, out var hosts);
if (hosts != null)
{
if (hosts.Contains(request.RequestUri.Host))
return true;
}

if (sslPolicyErrors != SslPolicyErrors.None)
{
// Throw exception with certificate details, this will cause a "Exception User-Unhandled" when running it in the Visual Studio debugger.
// The certificate is only available inside this function, so we can't catch it at the calling method.
throw new Exception("certificate validation failed: " + certificate.ToString());
}

return sslPolicyErrors == SslPolicyErrors.None;
}

public static void InitProxy(ServerConfig serverConfig)
{
// dispose old SocksWebProxy
if (webProxy is SocksWebProxy proxy)
proxy.Dispose();
webProxy = null;
webProxyUrl = serverConfig.GetProxyUrl();
if (!string.IsNullOrWhiteSpace(webProxyUrl))
{
if (serverConfig.ProxyType != ProxyType.Http)
{
var addresses = Dns.GetHostAddressesAsync(serverConfig.ProxyUrl).Result;
var socksConfig = new ProxyConfig
{
SocksAddress = addresses.FirstOrDefault(),
Username = serverConfig.ProxyUsername,
Password = serverConfig.ProxyPassword,
Version = serverConfig.ProxyType == ProxyType.Socks4 ?
ProxyConfig.SocksVersion.Four :
ProxyConfig.SocksVersion.Five
};
if (serverConfig.ProxyPort.HasValue)
{
socksConfig.SocksPort = serverConfig.ProxyPort.Value;
}
webProxy = new SocksWebProxy(socksConfig, false);
}
else
{
NetworkCredential creds = null;
if (!serverConfig.ProxyIsAnonymous)
{
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
creds = new NetworkCredential(username, password);
}
webProxy = new WebProxy(webProxyUrl)
{
BypassProxyOnLocal = false,
Credentials = creds
};
}
}
}

public HttpWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
: base(p: p,
l: l,
c: c,
sc: sc)
{
if (webProxyUrl == null)
InitProxy(sc);
}

// Called everytime the ServerConfig changes
public override void OnNext(ServerConfig value)
{
var newProxyUrl = serverConfig.GetProxyUrl();
if (webProxyUrl != newProxyUrl) // if proxy URL changed
InitProxy(serverConfig);
}

public override void Init()
{
ServicePointManager.DefaultConnectionLimit = 1000;

if (serverConfig.RuntimeSettings.IgnoreSslErrors == true)
{
logger.Info(string.Format("HttpWebClient: Disabling certificate validation"));
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => { return true; };
}

base.Init();
}

protected override async Task<WebClientByteResult> Run(WebRequest webRequest)
Expand Down Expand Up @@ -301,17 +209,5 @@ protected override async Task<WebClientByteResult> Run(WebRequest webRequest)
}
}
}

public override void AddTrustedCertificate(string host, string hash)
{
hash = hash.ToUpper();
trustedCertificates.TryGetValue(hash.ToUpper(), out var hosts);
if (hosts == null)
{
hosts = new HashSet<string>();
trustedCertificates[hash] = hosts;
}
hosts.Add(host);
}
}
}
105 changes: 3 additions & 102 deletions src/Jackett.Common/Utils/Clients/HttpWebClient2.cs
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,12 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CloudflareSolverRe;
using com.LandonKey.SocksWebProxy;
using com.LandonKey.SocksWebProxy.Proxy;
using Jackett.Common.Helpers;
using Jackett.Common.Models.Config;
using Jackett.Common.Services.Interfaces;
Expand All @@ -27,88 +22,13 @@ public class HttpWebClient2 : WebClient
private ClearanceHandler clearanceHandlr;
private HttpClientHandler clientHandlr;
private HttpClient client;

protected static Dictionary<string, ICollection<string>> trustedCertificates = new Dictionary<string, ICollection<string>>();
protected static string webProxyUrl;
protected static IWebProxy webProxy;

[DebuggerNonUserCode] // avoid "Exception User-Unhandled" Visual Studio messages
public static bool ValidateCertificate(HttpRequestMessage request, X509Certificate2 certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
{
var hash = certificate.GetCertHashString();


trustedCertificates.TryGetValue(hash, out var hosts);
if (hosts != null)
{
if (hosts.Contains(request.RequestUri.Host))
return true;
}

if (sslPolicyErrors != SslPolicyErrors.None)
{
// Throw exception with certificate details, this will cause a "Exception User-Unhandled" when running it in the Visual Studio debugger.
// The certificate is only available inside this function, so we can't catch it at the calling method.
throw new Exception("certificate validation failed: " + certificate.ToString());
}

return sslPolicyErrors == SslPolicyErrors.None;
}

public static void InitProxy(ServerConfig serverConfig)
{
// dispose old SocksWebProxy
if (webProxy is SocksWebProxy proxy)
proxy.Dispose();
webProxy = null;
webProxyUrl = serverConfig.GetProxyUrl();
if (!string.IsNullOrWhiteSpace(webProxyUrl))
{
if (serverConfig.ProxyType != ProxyType.Http)
{
var addresses = Dns.GetHostAddressesAsync(serverConfig.ProxyUrl).Result;
var socksConfig = new ProxyConfig
{
SocksAddress = addresses.FirstOrDefault(),
Username = serverConfig.ProxyUsername,
Password = serverConfig.ProxyPassword,
Version = serverConfig.ProxyType == ProxyType.Socks4 ?
ProxyConfig.SocksVersion.Four :
ProxyConfig.SocksVersion.Five
};
if (serverConfig.ProxyPort.HasValue)
{
socksConfig.SocksPort = serverConfig.ProxyPort.Value;
}
webProxy = new SocksWebProxy(socksConfig, false);
}
else
{
NetworkCredential creds = null;
if (!serverConfig.ProxyIsAnonymous)
{
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
creds = new NetworkCredential(username, password);
}
webProxy = new WebProxy(webProxyUrl)
{
BypassProxyOnLocal = false,
Credentials = creds
};
}
}
}


public HttpWebClient2(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
: base(p: p,
l: l,
c: c,
sc: sc)
{
if (webProxyUrl == null)
InitProxy(sc);

cookies = new CookieContainer();
CreateClient();
}
Expand Down Expand Up @@ -139,10 +59,7 @@ public void CreateClient()
// Called everytime the ServerConfig changes
public override void OnNext(ServerConfig value)
{
var newProxyUrl = serverConfig.GetProxyUrl();
if (webProxyUrl != newProxyUrl) // if proxy URL changed
InitProxy(serverConfig);

base.OnNext(value);
// recreate client if needed (can't just change the proxy attribute)
if (!ReferenceEquals(clientHandlr.Proxy, webProxy))
{
Expand All @@ -152,11 +69,7 @@ public override void OnNext(ServerConfig value)

public override void Init()
{
if (serverConfig.RuntimeSettings.IgnoreSslErrors == true)
{
logger.Info(string.Format("HttpWebClient2: Disabling certificate validation"));
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => { return true; };
}
base.Init();

ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
}
Expand Down Expand Up @@ -316,17 +229,5 @@ protected override async Task<WebClientByteResult> Run(WebRequest webRequest)
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return result;
}

public override void AddTrustedCertificate(string host, string hash)
{
hash = hash.ToUpper();
trustedCertificates.TryGetValue(hash.ToUpper(), out var hosts);
if (hosts == null)
{
hosts = new HashSet<string>();
trustedCertificates[hash] = hosts;
}
hosts.Add(host);
}
}
}

0 comments on commit c8b91be

Please sign in to comment.