poc: add embedded proxy to grab external cookies

azure-pipelines.yml

@@ -121,6 +121,13 @@ stages:
           zipAfterPublish: false
           arguments: '--configuration $(buildConfiguration) --runtime $(runtime) --framework $(framework) --output $(Build.BinariesDirectory) /p:AssemblyVersion=$(jackettVersion) /p:FileVersion=$(jackettVersion) /p:InformationalVersion=$(jackettVersion) /p:Version=$(jackettVersion)'
 
+      - task: CopyFiles@2
+        displayName: Copy Jackett CA
+        inputs:
+          SourceFolder: .
+          contents: 'jackettCA.*'
+          targetFolder: $(Build.BinariesDirectory)/Jackett
+
       - task: CopyFiles@2
         displayName: Copy Jackett Server
         inputs:
@@ -208,7 +215,7 @@ stages:
           script: |
             $file = '$(Build.BinariesDirectory)/Jackett/JackettConsole.exe.config'
             $xml = [xml] (Get-Content $file)
-            $newVersion = $xml.SelectSingleNode("configuration/runtime/*[name()='assemblyBinding']/*[name()='dependentAssembly']/*[name()='assemblyIdentity'][@name='System.Net.Http']/../*[name()='bindingRedirect']/@newVersion") 
+            $newVersion = $xml.SelectSingleNode("configuration/runtime/*[name()='assemblyBinding']/*[name()='dependentAssembly']/*[name()='assemblyIdentity'][@name='System.Net.Http']/../*[name()='bindingRedirect']/@newVersion")
             $newVersion.Value = '4.0.0.0'
             $xml.Save($file)
             Remove-Item '$(Build.BinariesDirectory)/Jackett/System.Net.Http.dll'
@@ -234,7 +241,7 @@ stages:
         displayName: Compress Binaries
         inputs:
           rootFolderOrFile: $(Build.BinariesDirectory)/Jackett
-          includeRootFolder: true 
+          includeRootFolder: true
           archiveType: '$(archiveType)'
           tarCompression: gz
           archiveFile: '$(Build.ArtifactStagingDirectory)/$(artifactName)'
@@ -245,9 +252,9 @@ stages:
         inputs:
           script: >
             iscc.exe $(Build.SourcesDirectory)/Installer.iss
-            /O"$(Build.ArtifactStagingDirectory)" 
-            /DMyFileForVersion=$(Build.BinariesDirectory)/Jackett/Jackett.Common.dll 
-            /DMySourceFolder=$(Build.BinariesDirectory)/Jackett 
+            /O"$(Build.ArtifactStagingDirectory)"
+            /DMyFileForVersion=$(Build.BinariesDirectory)/Jackett/Jackett.Common.dll
+            /DMySourceFolder=$(Build.BinariesDirectory)/Jackett
             /DMyOutputFilename=Jackett.Installer.Windows
 
       - task: PublishBuildArtifacts@1
@@ -495,7 +502,7 @@ stages:
         displayName: Create Github release
         inputs:
           gitHubConnection: JackettPublish
-          repositoryName: '$(Build.Repository.Name)' 
+          repositoryName: '$(Build.Repository.Name)'
           action: create
           target: $(Build.SourceVersion)
           tagSource: userSpecifiedTag

jackettCA.pem

@@ -0,0 +1,58 @@
+Bag Attributes
+    localKeyID: 22 40 E9 28 21 E8 31 B1 0D 23 EB EC D4 C2 AB 9B D9 96 BC CC 
+    localKeyID: 7A DE DA 4C CE 42 79 42 35 57 90 95 0B 12 E0 51 19 25 DB 92 
+    friendlyName: CN=Jackett Root Certificate Authority
+subject=CN = Jackett Root Certificate Authority
+
+issuer=CN = Jackett Root Certificate Authority
+
+-----BEGIN CERTIFICATE-----
+MIIDBDCCAeygAwIBAgIIVmKQhJpr9vAwDQYJKoZIhvcNAQELBQAwLTErMCkGA1UE
+AwwiSmFja2V0dCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0xOTAzMTQw
+MDUyNTFaFw0yNTAzMTMwMDUyNTFaMC0xKzApBgNVBAMMIkphY2tldHQgUm9vdCBD
+ZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCVIWLpzSb9eorXlL8r8ubsT80umfN5Zew7HWJ4ZU9CksZ01zasYEDSEIHn
+2sLUTEsXRwNoY+52f9KZdUDhuPF7AzOivHCi6qb/jovQS0Lw3bNPBU3IKjUqvItP
+r3LQxPovHXLKwcs7V4FMr3J/Z4D5kdG/T3E0hwAg3q0MJxCB4maYjxAHTnZDYEId
+vWeqpP8c/SZ0dfszvh6calD8R4qXKM+RYq1HiSxf367PFGSxWdcA5QbM/V+C94V2
++g6PCHJ9tjk2wxu/PoX73ZBf7O905S1EfxGVbwQY+yOabr4CxQXJTwIRGuwYJNbz
+8X6jAHw5icNHlHZv6wKz3v6CjznZAgMBAAGjKDAmMBMGA1UdJQQMMAoGCCsGAQUF
+BwMBMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEJZvBT7Zb6E
+uPOIXoZZ2M7FcMY943PkAE/T5UVeODT9rsS7BJGlG0Zl6EfrHsgmFo30vahHV7h+
+sCLMc5k1+QhuWMXE2B0FCBs9qTubBfoyUyg5ClLiLJSKW+QG6vGQ1G0JPmya61Lx
+1a0jG2qpRSzHrlVpC695hD59koQdKkK8se/Gw9+ZyoTGztgGvkRhmhUusokAW6cZ
+B4pn2UsYea9YMYfoeaOSlzBBYpO93Hxq5T3hQpayez5nCxSMZQrrB0hw58c7he5t
+7LGoPR9yLsj3vATvB7Bk8sglCd0miUz6ZyMNCH1WQk3hH3cn8VNF6MfgEGOe+wyW
+pmYVb3dZDIw=
+-----END CERTIFICATE-----
+Bag Attributes
+    localKeyID: 22 40 E9 28 21 E8 31 B1 0D 23 EB EC D4 C2 AB 9B D9 96 BC CC 
+Key Attributes: <No Attributes>
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCVIWLpzSb9eorX
+lL8r8ubsT80umfN5Zew7HWJ4ZU9CksZ01zasYEDSEIHn2sLUTEsXRwNoY+52f9KZ
+dUDhuPF7AzOivHCi6qb/jovQS0Lw3bNPBU3IKjUqvItPr3LQxPovHXLKwcs7V4FM
+r3J/Z4D5kdG/T3E0hwAg3q0MJxCB4maYjxAHTnZDYEIdvWeqpP8c/SZ0dfszvh6c
+alD8R4qXKM+RYq1HiSxf367PFGSxWdcA5QbM/V+C94V2+g6PCHJ9tjk2wxu/PoX7
+3ZBf7O905S1EfxGVbwQY+yOabr4CxQXJTwIRGuwYJNbz8X6jAHw5icNHlHZv6wKz
+3v6CjznZAgMBAAECggEAFPHp9wVhvwMZgfq5uN32MeVpX2yu5fN7MLhJTriH38VG
+iz14x9AC+p3n6NzwNSn79+p43439fXYpaXUu5iT4AXtrIqWNukvzpXvrRhdz8Olq
+WCRaDs1ixzxQ4qG1If4wVzKvHywFs7FwDwmrLpqmYibpSxHIyARX78XmjwjjiCJB
+t7iyVF8FZGhCOXlBlr/Wu6dBvHw0wxuJqO8SlxHdrI7MKUOgjMHzjcvjJgD2wPW8
+pBh2U4nZlHpeifa6yi/ye7TtG8jFTaPdyB98TSLESM/T1S3Q6iAuePEAIG6U7Ssy
+/9vQXzzyLlknFm1np7w0VghNcDD/n2wamMAqm0pMqwKBgQDI1v+vbHK8hqJGKKg5
+p8Yr0h3bTgnEsptTugzRABNxGT0VpQaKVkEoeLeMxH3asu6hRasCutAx6um6K7Uk
+I+ey/n8eiNNQJ0wd4Dm52WnMtQqS8cbuSj/L9vTU4g8kdsWngX6p0TcfHXBAhCuY
+udwAFrpPEc6Z2YExPW7Px4ftjwKBgQC+FrXawWWLABmKC3x5OtvkQiAgpVi5L8Mc
+Md/peVVF6RjU/zyPgwE5S8iyKy5fxs7YZbbaukxMBo3l350PcG761KFBdUWp9PJ2
+542+OlT7r0K6jr+WyWGD8kHoYtSRsmARTtBaKsyUW9vqkTCgTlsl+u/0IjyDz/N5
+vwHwFYv+FwKBgQCfPSh49Gl8ZNsg+Xd4Tzfm4q/dg+Bm3p4dInSq+X5wu+wczz2C
+TaVX627M47ZNwnVF1TEj9u6/xVwPyjvTLcy1tOchVKcG9EF7pp7eZi1mq1x46c4q
+fSBcHbA9YgdTiABrMVc2WbV0rCimXqUacLKoN1z8+Edw4G14lxbXE4MIrQKBgQCF
+riD8E7AendZYH5XNB9imYN6JNt77dRxokqaeYebXQINnH4xufLn6mlItAnMdhgnn
+YzjB/+wyNxXJiIXC2tYhbjFtgFSanpl5h4RGOG2Vhn6OBl+0YjFDArbY/8/wGpq2
+8U9Tae/vkd5JywuAYTp2rk/bx8v5AuB+SRbKycxmxwKBgBTquukc4vy+3Hp7usse
+BSvqswo/GqxfCQocZmVgrO26DsZIjgzKvmw+S6U5z4S3PgbaMQuoU//OLvdA6Nmk
+UK2HiXa2mxBMiFmZ6FhbmDy+6wUV/dtMFflL59EpK2zGIdsoMzvLe0NCnuTaKMQb
+qe/ec2QCOmKL+mAc5TVBGazM
+-----END PRIVATE KEY-----

src/Jackett.Server/Controllers/ProxyEventArgsBaseExtensions.cs

@@ -0,0 +1,17 @@
+using Titanium.Web.Proxy.EventArguments;
+
+namespace Jackett.Server.Controllers
+{
+    public static class ProxyEventArgsBaseExtensions
+    {
+        public static SampleClientState GetState(this ProxyEventArgsBase args)
+        {
+            if (args.ClientUserData == null)
+            {
+                args.ClientUserData = new SampleClientState();
+            }
+
+            return (SampleClientState)args.ClientUserData;
+        }
+    }
+}

src/Jackett.Server/Controllers/ProxyTestController.cs

@@ -0,0 +1,223 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Security.Cryptography.X509Certificates;
+using System.Threading;
+using System.Threading.Tasks;
+using Jackett.Common.Services.Interfaces;
+using Titanium.Web.Proxy;
+using Titanium.Web.Proxy.EventArguments;
+using Titanium.Web.Proxy.Exceptions;
+using Titanium.Web.Proxy.Helpers;
+using Titanium.Web.Proxy.Models;
+
+namespace Jackett.Server.Controllers
+{
+    public class ProxyTestController
+    {
+        private readonly SemaphoreSlim @lock = new SemaphoreSlim(1);
+        private readonly ProxyServer proxyServer;
+        private ExplicitProxyEndPoint explicitEndPoint;
+        private readonly IIndexerManagerService indexerService;
+        private readonly Dictionary<string,string> hostnameWhitelist;
+
+        public ProxyTestController(IIndexerManagerService i)
+        {
+            indexerService = i;
+
+            hostnameWhitelist = indexerService.GetAllIndexers().Select(x => new Uri(x.SiteLink).Host).ToList()
+                                              .Distinct().ToList().ToDictionary(x=> x, x => "");
+
+            proxyServer = new ProxyServer();
+
+            //proxyServer.EnableHttp2 = true;
+
+            // generate root certificate without storing it in file system
+/*
+            proxyServer.CertificateManager.PfxFilePath = "/home/xxx/repositories/jackett/jackettCA.pfx";
+            proxyServer.CertificateManager.PfxPassword = "123456";
+            proxyServer.CertificateManager.RootCertificateName = "Jackett Root Certificate Authority";
+            proxyServer.CertificateManager.RootCertificateIssuerName = "Jackett";
+            proxyServer.CertificateManager.CreateRootCertificate(true);
+
+            // convert to pem
+            openssl pkcs12 -in jackettCA.pfx -out jackettCA.pem -nodes
+*/
+
+
+            //proxyServer.CertificateManager.TrustRootCertificate();
+            //proxyServer.CertificateManager.TrustRootCertificateAsAdmin();
+
+            proxyServer.ExceptionFunc = async exception =>
+            {
+                if (exception is ProxyHttpException phex)
+                {
+                    await writeToConsole(exception.Message + ": " + phex.InnerException?.Message, ConsoleColor.Red);
+                }
+                else
+                {
+                    await writeToConsole(exception.Message, ConsoleColor.Red);
+                }
+            };
+
+            proxyServer.TcpTimeWaitSeconds = 10;
+            proxyServer.ConnectionTimeOutSeconds = 15;
+            proxyServer.ReuseSocket = false;
+            proxyServer.EnableConnectionPool = false;
+            proxyServer.ForwardToUpstreamGateway = true;
+            proxyServer.CertificateManager.SaveFakeCertificates = false;
+            //proxyServer.ProxyBasicAuthenticateFunc = async (args, userName, password) =>
+            //{
+            //    return true;
+            //};
+
+            // this is just to show the functionality, provided implementations use junk value
+            //proxyServer.GetCustomUpStreamProxyFunc = onGetCustomUpStreamProxyFunc;
+            //proxyServer.CustomUpStreamProxyFailureFunc = onCustomUpStreamProxyFailureFunc;
+
+            // optionally set the Certificate Engine
+            // Under Mono or Non-Windows runtimes only BouncyCastle will be supported
+            //proxyServer.CertificateManager.CertificateEngine = Network.CertificateEngine.BouncyCastle;
+
+            // optionally set the Root Certificate
+            var pathCA = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "jackettCA.pfx");
+            proxyServer.CertificateManager.RootCertificate = new X509Certificate2(pathCA, "123456", X509KeyStorageFlags.Exportable);
+        }
+
+        public void StartProxy()
+        {
+            proxyServer.BeforeRequest += onRequest;
+            //proxyServer.BeforeResponse += onResponse;
+            //proxyServer.AfterResponse += onAfterResponse;
+
+            //proxyServer.ServerCertificateValidationCallback += OnCertificateValidation;
+            //proxyServer.ClientCertificateSelectionCallback += OnCertificateSelection;
+
+            //proxyServer.EnableWinAuth = true;
+
+            explicitEndPoint = new ExplicitProxyEndPoint(IPAddress.Any, 8000);
+
+            // Fired when a CONNECT request is received
+            explicitEndPoint.BeforeTunnelConnectRequest += onBeforeTunnelConnectRequest;
+            //explicitEndPoint.BeforeTunnelConnectResponse += onBeforeTunnelConnectResponse;
+
+            // An explicit endpoint is where the client knows about the existence of a proxy
+            // So client sends request in a proxy friendly manner
+            proxyServer.AddEndPoint(explicitEndPoint);
+            proxyServer.Start();
+
+            // Transparent endpoint is useful for reverse proxy (client is not aware of the existence of proxy)
+            // A transparent endpoint usually requires a network router port forwarding HTTP(S) packets or DNS
+            // to send data to this endPoint
+            //var transparentEndPoint = new TransparentProxyEndPoint(IPAddress.Any, 443, true)
+            //{
+            //    // Generic Certificate hostname to use
+            //    // When SNI is disabled by client
+            //    GenericCertificateName = "google.com"
+            //};
+
+            //proxyServer.AddEndPoint(transparentEndPoint);
+            //proxyServer.UpStreamHttpProxy = new ExternalProxy("localhost", 8888);
+            //proxyServer.UpStreamHttpsProxy = new ExternalProxy("localhost", 8888);
+
+            // SOCKS proxy
+            //proxyServer.UpStreamHttpProxy = new ExternalProxy("127.0.0.1", 1080)
+            //    { ProxyType = ExternalProxyType.Socks5, UserName = "User1", Password = "Pass" };
+            //proxyServer.UpStreamHttpsProxy = new ExternalProxy("127.0.0.1", 1080)
+            //    { ProxyType = ExternalProxyType.Socks5, UserName = "User1", Password = "Pass" };
+
+
+            //var socksEndPoint = new SocksProxyEndPoint(IPAddress.Any, 1080, true)
+            //{
+            //    // Generic Certificate hostname to use
+            //    // When SNI is disabled by client
+            //    GenericCertificateName = "google.com"
+            //};
+
+            //proxyServer.AddEndPoint(socksEndPoint);
+
+            foreach (var endPoint in proxyServer.ProxyEndPoints)
+            {
+                Console.WriteLine("Listening on '{0}' endpoint at Ip {1} and port: {2} ", endPoint.GetType().Name,
+                    endPoint.IpAddress, endPoint.Port);
+            }
+
+            // Only explicit proxies can be set as system proxy!
+            //proxyServer.SetAsSystemHttpProxy(explicitEndPoint);
+            //proxyServer.SetAsSystemHttpsProxy(explicitEndPoint);
+            if (RunTime.IsWindows)
+            {
+                proxyServer.SetAsSystemProxy(explicitEndPoint, ProxyProtocolType.AllHttp);
+            }
+        }
+
+        private async Task onBeforeTunnelConnectRequest(object sender, TunnelConnectSessionEventArgs e)
+        {
+            var hostname = e.HttpClient.Request.RequestUri.Host;
+            //e.GetState().PipelineInfo.AppendLine(nameof(onBeforeTunnelConnectRequest) + ":" + hostname);
+            //await writeToConsole("Tunnel to: " + hostname);
+
+            var clientLocalIp = e.ClientLocalEndPoint.Address;
+            if (!clientLocalIp.Equals(IPAddress.Loopback) && !clientLocalIp.Equals(IPAddress.IPv6Loopback))
+            {
+                e.HttpClient.UpStreamEndPoint = new IPEndPoint(clientLocalIp, 0);
+            }
+
+            e.DecryptSsl = false;
+            foreach (var item in hostnameWhitelist)
+            {
+                if (hostname.Equals(item.Key))
+                {
+                    e.DecryptSsl = true;
+                    break;
+                }
+            }
+        }
+
+        // intercept & cancel redirect or update requests
+        private async Task onRequest(object sender, SessionEventArgs e)
+        {
+            //e.GetState().PipelineInfo.AppendLine(nameof(onRequest) + ":" + e.HttpClient.Request.RequestUri);
+
+            var clientLocalIp = e.ClientLocalEndPoint.Address;
+            if (!clientLocalIp.Equals(IPAddress.Loopback) && !clientLocalIp.Equals(IPAddress.IPv6Loopback))
+            {
+                e.HttpClient.UpStreamEndPoint = new IPEndPoint(clientLocalIp, 0);
+            }
+
+            var host = e.HttpClient.Request.Host;
+            if (hostnameWhitelist.ContainsKey(host))
+            {
+                var cookie = e.HttpClient.Request.Headers.GetHeaders("Cookie")?.First().Value;
+                if (!hostnameWhitelist[host].Equals(cookie))
+                {
+                    await writeToConsole("host: " + host + " cookie: " + cookie);
+                    hostnameWhitelist[host] = cookie;
+                }
+            }
+        }
+
+        private async Task writeToConsole(string message, ConsoleColor? consoleColor = null)
+        {
+            await @lock.WaitAsync();
+
+            if (consoleColor.HasValue)
+            {
+                ConsoleColor existing = Console.ForegroundColor;
+                Console.ForegroundColor = consoleColor.Value;
+                Console.WriteLine(message);
+                Console.ForegroundColor = existing;
+            }
+            else
+            {
+                Console.WriteLine(message);
+            }
+
+            @lock.Release();
+        }
+
+    }
+}
+

src/Jackett.Server/Controllers/SampleClientState.cs

@@ -0,0 +1,9 @@
+using System.Text;
+
+namespace Jackett.Server.Controllers
+{
+    public class SampleClientState
+    {
+        public StringBuilder PipelineInfo { get; } = new StringBuilder();
+    }
+}

src/Jackett.Server/Jackett.Server.csproj

@@ -60,6 +60,7 @@
     <PackageReference Include="NLog.Web.AspNetCore" Version="4.9.0" />
     <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.7.0" />
     <PackageReference Include="System.Text.Encoding.CodePages" Version="4.7.0" />
+    <PackageReference Include="Titanium.Web.Proxy" Version="3.1.1276" />
   </ItemGroup>
 
   <ItemGroup>
@@ -73,6 +74,12 @@
     <Content Include="..\..\LICENSE">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="..\..\jackettCA.pfx">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="..\..\jackettCA.pem">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
 
 </Project>