diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d8c4e2a44adce..008235c52cedd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -111,6 +111,13 @@ stages: zipAfterPublish: false arguments: '--configuration $(buildConfiguration) --runtime $(runtime) --framework $(framework) --output $(Build.BinariesDirectory) /p:AssemblyVersion=$(jackettVersion) /p:FileVersion=$(jackettVersion) /p:InformationalVersion=$(jackettVersion) /p:Version=$(jackettVersion)' + - task: CopyFiles@2 + displayName: Copy Jackett CA + inputs: + SourceFolder: . + contents: 'jackettCA.*' + targetFolder: $(Build.BinariesDirectory)/Jackett + - task: CopyFiles@2 displayName: Copy Jackett Server inputs: @@ -198,7 +205,7 @@ stages: script: | $file = '$(Build.BinariesDirectory)/Jackett/JackettConsole.exe.config' $xml = [xml] (Get-Content $file) - $newVersion = $xml.SelectSingleNode("configuration/runtime/*[name()='assemblyBinding']/*[name()='dependentAssembly']/*[name()='assemblyIdentity'][@name='System.Net.Http']/../*[name()='bindingRedirect']/@newVersion") + $newVersion = $xml.SelectSingleNode("configuration/runtime/*[name()='assemblyBinding']/*[name()='dependentAssembly']/*[name()='assemblyIdentity'][@name='System.Net.Http']/../*[name()='bindingRedirect']/@newVersion") $newVersion.Value = '4.0.0.0' $xml.Save($file) Remove-Item '$(Build.BinariesDirectory)/Jackett/System.Net.Http.dll' @@ -224,7 +231,7 @@ stages: displayName: Compress Binaries inputs: rootFolderOrFile: $(Build.BinariesDirectory)/Jackett - includeRootFolder: true + includeRootFolder: true archiveType: '$(archiveType)' tarCompression: gz archiveFile: '$(Build.ArtifactStagingDirectory)/$(artifactName)' @@ -235,9 +242,9 @@ stages: inputs: script: > iscc.exe $(Build.SourcesDirectory)/Installer.iss - /O"$(Build.ArtifactStagingDirectory)" - /DMyFileForVersion=$(Build.BinariesDirectory)/Jackett/Jackett.Common.dll - /DMySourceFolder=$(Build.BinariesDirectory)/Jackett + /O"$(Build.ArtifactStagingDirectory)" + /DMyFileForVersion=$(Build.BinariesDirectory)/Jackett/Jackett.Common.dll + /DMySourceFolder=$(Build.BinariesDirectory)/Jackett /DMyOutputFilename=Jackett.Installer.Windows - task: PublishBuildArtifacts@1 @@ -357,7 +364,7 @@ stages: displayName: Create Github release inputs: gitHubConnection: github.com_jackett - repositoryName: '$(Build.Repository.Name)' + repositoryName: '$(Build.Repository.Name)' action: create target: $(Build.SourceVersion) tagSource: manual @@ -379,12 +386,12 @@ stages: { Write-Output $logUrl $logText = Invoke-WebRequest $logUrl - if ($logText -like '*: GitHub Release*') + if ($logText -like '*: GitHub Release*') { $successCount = (Select-String "Uploaded file successfully:" -InputObject $logText -AllMatches).Matches.Count $failureCount = (Select-String "Duplicate asset found:" -InputObject $logText -AllMatches).Matches.Count Write-Host "Success count is: $successCount and failure count is: $failureCount" - if (($successCount -ne 7) -or ($failureCount -ne 0)) { Write-Host "##vso[task.complete result=Failed;]DONE" } + if (($successCount -ne 7) -or ($failureCount -ne 0)) { Write-Host "##vso[task.complete result=Failed;]DONE" } } } diff --git a/jackettCA.pem b/jackettCA.pem new file mode 100644 index 0000000000000..93ad0fbff7416 --- /dev/null +++ b/jackettCA.pem @@ -0,0 +1,58 @@ +Bag Attributes + localKeyID: 22 40 E9 28 21 E8 31 B1 0D 23 EB EC D4 C2 AB 9B D9 96 BC CC + localKeyID: 7A DE DA 4C CE 42 79 42 35 57 90 95 0B 12 E0 51 19 25 DB 92 + friendlyName: CN=Jackett Root Certificate Authority +subject=CN = Jackett Root Certificate Authority + +issuer=CN = Jackett Root Certificate Authority + +-----BEGIN CERTIFICATE----- +MIIDBDCCAeygAwIBAgIIVmKQhJpr9vAwDQYJKoZIhvcNAQELBQAwLTErMCkGA1UE +AwwiSmFja2V0dCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0xOTAzMTQw +MDUyNTFaFw0yNTAzMTMwMDUyNTFaMC0xKzApBgNVBAMMIkphY2tldHQgUm9vdCBD +ZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCVIWLpzSb9eorXlL8r8ubsT80umfN5Zew7HWJ4ZU9CksZ01zasYEDSEIHn +2sLUTEsXRwNoY+52f9KZdUDhuPF7AzOivHCi6qb/jovQS0Lw3bNPBU3IKjUqvItP +r3LQxPovHXLKwcs7V4FMr3J/Z4D5kdG/T3E0hwAg3q0MJxCB4maYjxAHTnZDYEId +vWeqpP8c/SZ0dfszvh6calD8R4qXKM+RYq1HiSxf367PFGSxWdcA5QbM/V+C94V2 ++g6PCHJ9tjk2wxu/PoX73ZBf7O905S1EfxGVbwQY+yOabr4CxQXJTwIRGuwYJNbz +8X6jAHw5icNHlHZv6wKz3v6CjznZAgMBAAGjKDAmMBMGA1UdJQQMMAoGCCsGAQUF +BwMBMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEJZvBT7Zb6E +uPOIXoZZ2M7FcMY943PkAE/T5UVeODT9rsS7BJGlG0Zl6EfrHsgmFo30vahHV7h+ +sCLMc5k1+QhuWMXE2B0FCBs9qTubBfoyUyg5ClLiLJSKW+QG6vGQ1G0JPmya61Lx +1a0jG2qpRSzHrlVpC695hD59koQdKkK8se/Gw9+ZyoTGztgGvkRhmhUusokAW6cZ +B4pn2UsYea9YMYfoeaOSlzBBYpO93Hxq5T3hQpayez5nCxSMZQrrB0hw58c7he5t +7LGoPR9yLsj3vATvB7Bk8sglCd0miUz6ZyMNCH1WQk3hH3cn8VNF6MfgEGOe+wyW +pmYVb3dZDIw= +-----END CERTIFICATE----- +Bag Attributes + localKeyID: 22 40 E9 28 21 E8 31 B1 0D 23 EB EC D4 C2 AB 9B D9 96 BC CC +Key Attributes: +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCVIWLpzSb9eorX +lL8r8ubsT80umfN5Zew7HWJ4ZU9CksZ01zasYEDSEIHn2sLUTEsXRwNoY+52f9KZ +dUDhuPF7AzOivHCi6qb/jovQS0Lw3bNPBU3IKjUqvItPr3LQxPovHXLKwcs7V4FM +r3J/Z4D5kdG/T3E0hwAg3q0MJxCB4maYjxAHTnZDYEIdvWeqpP8c/SZ0dfszvh6c +alD8R4qXKM+RYq1HiSxf367PFGSxWdcA5QbM/V+C94V2+g6PCHJ9tjk2wxu/PoX7 +3ZBf7O905S1EfxGVbwQY+yOabr4CxQXJTwIRGuwYJNbz8X6jAHw5icNHlHZv6wKz +3v6CjznZAgMBAAECggEAFPHp9wVhvwMZgfq5uN32MeVpX2yu5fN7MLhJTriH38VG +iz14x9AC+p3n6NzwNSn79+p43439fXYpaXUu5iT4AXtrIqWNukvzpXvrRhdz8Olq +WCRaDs1ixzxQ4qG1If4wVzKvHywFs7FwDwmrLpqmYibpSxHIyARX78XmjwjjiCJB +t7iyVF8FZGhCOXlBlr/Wu6dBvHw0wxuJqO8SlxHdrI7MKUOgjMHzjcvjJgD2wPW8 +pBh2U4nZlHpeifa6yi/ye7TtG8jFTaPdyB98TSLESM/T1S3Q6iAuePEAIG6U7Ssy +/9vQXzzyLlknFm1np7w0VghNcDD/n2wamMAqm0pMqwKBgQDI1v+vbHK8hqJGKKg5 +p8Yr0h3bTgnEsptTugzRABNxGT0VpQaKVkEoeLeMxH3asu6hRasCutAx6um6K7Uk +I+ey/n8eiNNQJ0wd4Dm52WnMtQqS8cbuSj/L9vTU4g8kdsWngX6p0TcfHXBAhCuY +udwAFrpPEc6Z2YExPW7Px4ftjwKBgQC+FrXawWWLABmKC3x5OtvkQiAgpVi5L8Mc +Md/peVVF6RjU/zyPgwE5S8iyKy5fxs7YZbbaukxMBo3l350PcG761KFBdUWp9PJ2 +542+OlT7r0K6jr+WyWGD8kHoYtSRsmARTtBaKsyUW9vqkTCgTlsl+u/0IjyDz/N5 +vwHwFYv+FwKBgQCfPSh49Gl8ZNsg+Xd4Tzfm4q/dg+Bm3p4dInSq+X5wu+wczz2C +TaVX627M47ZNwnVF1TEj9u6/xVwPyjvTLcy1tOchVKcG9EF7pp7eZi1mq1x46c4q +fSBcHbA9YgdTiABrMVc2WbV0rCimXqUacLKoN1z8+Edw4G14lxbXE4MIrQKBgQCF +riD8E7AendZYH5XNB9imYN6JNt77dRxokqaeYebXQINnH4xufLn6mlItAnMdhgnn +YzjB/+wyNxXJiIXC2tYhbjFtgFSanpl5h4RGOG2Vhn6OBl+0YjFDArbY/8/wGpq2 +8U9Tae/vkd5JywuAYTp2rk/bx8v5AuB+SRbKycxmxwKBgBTquukc4vy+3Hp7usse +BSvqswo/GqxfCQocZmVgrO26DsZIjgzKvmw+S6U5z4S3PgbaMQuoU//OLvdA6Nmk +UK2HiXa2mxBMiFmZ6FhbmDy+6wUV/dtMFflL59EpK2zGIdsoMzvLe0NCnuTaKMQb +qe/ec2QCOmKL+mAc5TVBGazM +-----END PRIVATE KEY----- diff --git a/jackettCA.pfx b/jackettCA.pfx new file mode 100644 index 0000000000000..6ca66e4968653 Binary files /dev/null and b/jackettCA.pfx differ diff --git a/src/Jackett.Server/Controllers/ProxyEventArgsBaseExtensions.cs b/src/Jackett.Server/Controllers/ProxyEventArgsBaseExtensions.cs new file mode 100644 index 0000000000000..402c18fe0b78a --- /dev/null +++ b/src/Jackett.Server/Controllers/ProxyEventArgsBaseExtensions.cs @@ -0,0 +1,17 @@ +using Titanium.Web.Proxy.EventArguments; + +namespace Jackett.Server.Controllers +{ + public static class ProxyEventArgsBaseExtensions + { + public static SampleClientState GetState(this ProxyEventArgsBase args) + { + if (args.ClientUserData == null) + { + args.ClientUserData = new SampleClientState(); + } + + return (SampleClientState)args.ClientUserData; + } + } +} diff --git a/src/Jackett.Server/Controllers/ProxyTestController.cs b/src/Jackett.Server/Controllers/ProxyTestController.cs new file mode 100644 index 0000000000000..073a11d755d35 --- /dev/null +++ b/src/Jackett.Server/Controllers/ProxyTestController.cs @@ -0,0 +1,223 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Jackett.Common.Services.Interfaces; +using Titanium.Web.Proxy; +using Titanium.Web.Proxy.EventArguments; +using Titanium.Web.Proxy.Exceptions; +using Titanium.Web.Proxy.Helpers; +using Titanium.Web.Proxy.Models; + +namespace Jackett.Server.Controllers +{ + public class ProxyTestController + { + private readonly SemaphoreSlim @lock = new SemaphoreSlim(1); + private readonly ProxyServer proxyServer; + private ExplicitProxyEndPoint explicitEndPoint; + private readonly IIndexerManagerService indexerService; + private readonly Dictionary hostnameWhitelist; + + public ProxyTestController(IIndexerManagerService i) + { + indexerService = i; + + hostnameWhitelist = indexerService.GetAllIndexers().Select(x => new Uri(x.SiteLink).Host).ToList() + .Distinct().ToList().ToDictionary(x=> x, x => ""); + + proxyServer = new ProxyServer(); + + //proxyServer.EnableHttp2 = true; + + // generate root certificate without storing it in file system +/* + proxyServer.CertificateManager.PfxFilePath = "/home/xxx/repositories/jackett/jackettCA.pfx"; + proxyServer.CertificateManager.PfxPassword = "123456"; + proxyServer.CertificateManager.RootCertificateName = "Jackett Root Certificate Authority"; + proxyServer.CertificateManager.RootCertificateIssuerName = "Jackett"; + proxyServer.CertificateManager.CreateRootCertificate(true); + + // convert to pem + openssl pkcs12 -in jackettCA.pfx -out jackettCA.pem -nodes +*/ + + + //proxyServer.CertificateManager.TrustRootCertificate(); + //proxyServer.CertificateManager.TrustRootCertificateAsAdmin(); + + proxyServer.ExceptionFunc = async exception => + { + if (exception is ProxyHttpException phex) + { + await writeToConsole(exception.Message + ": " + phex.InnerException?.Message, ConsoleColor.Red); + } + else + { + await writeToConsole(exception.Message, ConsoleColor.Red); + } + }; + + proxyServer.TcpTimeWaitSeconds = 10; + proxyServer.ConnectionTimeOutSeconds = 15; + proxyServer.ReuseSocket = false; + proxyServer.EnableConnectionPool = false; + proxyServer.ForwardToUpstreamGateway = true; + proxyServer.CertificateManager.SaveFakeCertificates = false; + //proxyServer.ProxyBasicAuthenticateFunc = async (args, userName, password) => + //{ + // return true; + //}; + + // this is just to show the functionality, provided implementations use junk value + //proxyServer.GetCustomUpStreamProxyFunc = onGetCustomUpStreamProxyFunc; + //proxyServer.CustomUpStreamProxyFailureFunc = onCustomUpStreamProxyFailureFunc; + + // optionally set the Certificate Engine + // Under Mono or Non-Windows runtimes only BouncyCastle will be supported + //proxyServer.CertificateManager.CertificateEngine = Network.CertificateEngine.BouncyCastle; + + // optionally set the Root Certificate + var pathCA = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "jackettCA.pfx"); + proxyServer.CertificateManager.RootCertificate = new X509Certificate2(pathCA, "123456", X509KeyStorageFlags.Exportable); + } + + public void StartProxy() + { + proxyServer.BeforeRequest += onRequest; + //proxyServer.BeforeResponse += onResponse; + //proxyServer.AfterResponse += onAfterResponse; + + //proxyServer.ServerCertificateValidationCallback += OnCertificateValidation; + //proxyServer.ClientCertificateSelectionCallback += OnCertificateSelection; + + //proxyServer.EnableWinAuth = true; + + explicitEndPoint = new ExplicitProxyEndPoint(IPAddress.Any, 8000); + + // Fired when a CONNECT request is received + explicitEndPoint.BeforeTunnelConnectRequest += onBeforeTunnelConnectRequest; + //explicitEndPoint.BeforeTunnelConnectResponse += onBeforeTunnelConnectResponse; + + // An explicit endpoint is where the client knows about the existence of a proxy + // So client sends request in a proxy friendly manner + proxyServer.AddEndPoint(explicitEndPoint); + proxyServer.Start(); + + // Transparent endpoint is useful for reverse proxy (client is not aware of the existence of proxy) + // A transparent endpoint usually requires a network router port forwarding HTTP(S) packets or DNS + // to send data to this endPoint + //var transparentEndPoint = new TransparentProxyEndPoint(IPAddress.Any, 443, true) + //{ + // // Generic Certificate hostname to use + // // When SNI is disabled by client + // GenericCertificateName = "google.com" + //}; + + //proxyServer.AddEndPoint(transparentEndPoint); + //proxyServer.UpStreamHttpProxy = new ExternalProxy("localhost", 8888); + //proxyServer.UpStreamHttpsProxy = new ExternalProxy("localhost", 8888); + + // SOCKS proxy + //proxyServer.UpStreamHttpProxy = new ExternalProxy("127.0.0.1", 1080) + // { ProxyType = ExternalProxyType.Socks5, UserName = "User1", Password = "Pass" }; + //proxyServer.UpStreamHttpsProxy = new ExternalProxy("127.0.0.1", 1080) + // { ProxyType = ExternalProxyType.Socks5, UserName = "User1", Password = "Pass" }; + + + //var socksEndPoint = new SocksProxyEndPoint(IPAddress.Any, 1080, true) + //{ + // // Generic Certificate hostname to use + // // When SNI is disabled by client + // GenericCertificateName = "google.com" + //}; + + //proxyServer.AddEndPoint(socksEndPoint); + + foreach (var endPoint in proxyServer.ProxyEndPoints) + { + Console.WriteLine("Listening on '{0}' endpoint at Ip {1} and port: {2} ", endPoint.GetType().Name, + endPoint.IpAddress, endPoint.Port); + } + + // Only explicit proxies can be set as system proxy! + //proxyServer.SetAsSystemHttpProxy(explicitEndPoint); + //proxyServer.SetAsSystemHttpsProxy(explicitEndPoint); + if (RunTime.IsWindows) + { + proxyServer.SetAsSystemProxy(explicitEndPoint, ProxyProtocolType.AllHttp); + } + } + + private async Task onBeforeTunnelConnectRequest(object sender, TunnelConnectSessionEventArgs e) + { + var hostname = e.HttpClient.Request.RequestUri.Host; + //e.GetState().PipelineInfo.AppendLine(nameof(onBeforeTunnelConnectRequest) + ":" + hostname); + //await writeToConsole("Tunnel to: " + hostname); + + var clientLocalIp = e.ClientLocalEndPoint.Address; + if (!clientLocalIp.Equals(IPAddress.Loopback) && !clientLocalIp.Equals(IPAddress.IPv6Loopback)) + { + e.HttpClient.UpStreamEndPoint = new IPEndPoint(clientLocalIp, 0); + } + + e.DecryptSsl = false; + foreach (var item in hostnameWhitelist) + { + if (hostname.Equals(item.Key)) + { + e.DecryptSsl = true; + break; + } + } + } + + // intercept & cancel redirect or update requests + private async Task onRequest(object sender, SessionEventArgs e) + { + //e.GetState().PipelineInfo.AppendLine(nameof(onRequest) + ":" + e.HttpClient.Request.RequestUri); + + var clientLocalIp = e.ClientLocalEndPoint.Address; + if (!clientLocalIp.Equals(IPAddress.Loopback) && !clientLocalIp.Equals(IPAddress.IPv6Loopback)) + { + e.HttpClient.UpStreamEndPoint = new IPEndPoint(clientLocalIp, 0); + } + + var host = e.HttpClient.Request.Host; + if (hostnameWhitelist.ContainsKey(host)) + { + var cookie = e.HttpClient.Request.Headers.GetHeaders("Cookie")?.First().Value; + if (!hostnameWhitelist[host].Equals(cookie)) + { + await writeToConsole("host: " + host + " cookie: " + cookie); + hostnameWhitelist[host] = cookie; + } + } + } + + private async Task writeToConsole(string message, ConsoleColor? consoleColor = null) + { + await @lock.WaitAsync(); + + if (consoleColor.HasValue) + { + ConsoleColor existing = Console.ForegroundColor; + Console.ForegroundColor = consoleColor.Value; + Console.WriteLine(message); + Console.ForegroundColor = existing; + } + else + { + Console.WriteLine(message); + } + + @lock.Release(); + } + + } +} + diff --git a/src/Jackett.Server/Controllers/SampleClientState.cs b/src/Jackett.Server/Controllers/SampleClientState.cs new file mode 100644 index 0000000000000..2cfb1d904a2e2 --- /dev/null +++ b/src/Jackett.Server/Controllers/SampleClientState.cs @@ -0,0 +1,9 @@ +using System.Text; + +namespace Jackett.Server.Controllers +{ + public class SampleClientState + { + public StringBuilder PipelineInfo { get; } = new StringBuilder(); + } +} diff --git a/src/Jackett.Server/Jackett.Server.csproj b/src/Jackett.Server/Jackett.Server.csproj index 7a1c64b56b68c..a7fbf4d6fc69f 100644 --- a/src/Jackett.Server/Jackett.Server.csproj +++ b/src/Jackett.Server/Jackett.Server.csproj @@ -60,6 +60,7 @@ + @@ -73,6 +74,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + diff --git a/src/Jackett.Server/Services/ServerService.cs b/src/Jackett.Server/Services/ServerService.cs index a53f0a5e9cacc..8565e30113c77 100644 --- a/src/Jackett.Server/Services/ServerService.cs +++ b/src/Jackett.Server/Services/ServerService.cs @@ -14,6 +14,7 @@ using Jackett.Common.Models.Config; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Server.Controllers; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.WebUtilities; using NLog; @@ -335,6 +336,11 @@ public void Initalize() CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US"); // Load indexers indexerService.InitIndexers(configService.GetCardigannDefinitionsFolders()); + + var controller = new ProxyTestController(indexerService); + // Start proxy controller + controller.StartProxy(); + client.Init(); updater.CleanupTempDir(); }