FlareSolverr.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using FluentValidation.Results;
using Newtonsoft.Json;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Cloud;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Http.Proxy;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.Localization;
using NzbDrone.Core.Validation;

namespace NzbDrone.Core.IndexerProxies.FlareSolverr
{
    public class FlareSolverr : HttpIndexerProxyBase<FlareSolverrSettings>
    {
        private readonly ICached<string> _cache;
        private readonly IHttpProxySettingsProvider _proxySettingsProvider;

        public FlareSolverr(IHttpProxySettingsProvider proxySettingsProvider, IProwlarrCloudRequestBuilder cloudRequestBuilder, IHttpClient httpClient, Logger logger, ILocalizationService localizationService, ICacheManager cacheManager)
            : base(cloudRequestBuilder, httpClient, logger, localizationService)
        {
            _proxySettingsProvider = proxySettingsProvider;
            _cache = cacheManager.GetCache<string>(typeof(string), "UserAgent");
        }

        public override string Name => "FlareSolverr";

        public override HttpRequest PreRequest(HttpRequest request)
        {
            //Try original request first, ignore errors, detect CF in post response
            request.SuppressHttpError = true;

            //Inject UA if not present
            if (_cache.Find(request.Url.Host).IsNotNullOrWhiteSpace() && request.Headers.UserAgent.IsNullOrWhiteSpace())
            {
                request.Headers.UserAgent = _cache.Find(request.Url.Host);
            }

            return request;
        }

        public override HttpResponse PostResponse(HttpResponse response)
        {
            if (!CloudFlareDetectionService.IsCloudflareProtected(response))
            {
                _logger.Debug("CF Protection not detected, returning original response");
                return response;
            }

            var flaresolverrResponse = _httpClient.Execute(GenerateFlareSolverrRequest(response.Request));

            if (flaresolverrResponse.StatusCode != HttpStatusCode.OK && flaresolverrResponse.StatusCode != HttpStatusCode.InternalServerError)
            {
                throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :" + response.StatusCode);
            }

            var result = JsonConvert.DeserializeObject<FlareSolverrResponse>(flaresolverrResponse.Content);

            var newRequest = response.Request;

            //Cache the user-agent so we can inject it in next request to avoid re-solve
            _cache.Set(response.Request.Url.Host, result.Solution.UserAgent);
            newRequest.Headers.UserAgent = result.Solution.UserAgent;

            InjectCookies(newRequest, result);

            //Request again with User-Agent and Cookies from Flaresolverr
            var finalResponse = _httpClient.Execute(newRequest);

            return finalResponse;
        }

        private void InjectCookies(HttpRequest request, FlareSolverrResponse flareSolverrResponse)
        {
            var rCookies = flareSolverrResponse.Solution.Cookies;

            if (!rCookies.Any())
            {
                return;
            }

            var rCookiesList = rCookies.Select(x => x.Name).ToList();

            foreach (var rCookie in rCookies)
            {
                request.Cookies.Remove(rCookie.Name);
                request.Cookies.Add(rCookie.Name, rCookie.Value);
            }
        }

        private HttpRequest GenerateFlareSolverrRequest(HttpRequest request)
        {
            FlareSolverrRequest req;

            var url = request.Url.ToString();
            var maxTimeout = Settings.RequestTimeout * 1000;

            var proxySettings = _proxySettingsProvider.GetProxySettings();
            var proxyUrl = proxySettings != null ? GetProxyUri(proxySettings) : null;

            var requestProxy = new FlareSolverrProxy
            {
                Url = proxyUrl?.OriginalString,
                Username = proxySettings != null && proxySettings.Username.IsNotNullOrWhiteSpace() ? proxySettings.Username : null,
                Password = proxySettings != null && proxySettings.Password.IsNotNullOrWhiteSpace() ? proxySettings.Password : null
            };

            if (request.Method == HttpMethod.Get)
            {
                req = new FlareSolverrRequestGet
                {
                    Cmd = "request.get",
                    Url = url,
                    MaxTimeout = maxTimeout,
                    Proxy = requestProxy
                };
            }
            else if (request.Method == HttpMethod.Post)
            {
                var contentTypeType = request.Headers.ContentType.ToLower() ?? "<null>";

                if (contentTypeType.Contains("application/x-www-form-urlencoded"))
                {
                    var contentTypeValue = request.Headers.ContentType.ToString();
                    var postData = request.GetContent();

                    req = new FlareSolverrRequestPostUrlEncoded
                    {
                        Cmd = "request.post",
                        Url = url,
                        PostData = postData,
                        Headers = new HeadersPost
                        {
                            ContentType = contentTypeValue,
                            ContentLength = null
                        },
                        MaxTimeout = maxTimeout,
                        Proxy = requestProxy
                    };
                }
                else if (contentTypeType.Contains("multipart/form-data")
                         || contentTypeType.Contains("text/html"))
                {
                    //TODO Implement - check if we just need to pass the content-type with the relevant headers
                    throw new FlareSolverrException("Unimplemented POST Content-Type: " + request.Headers.ContentType);
                }
                else
                {
                    throw new FlareSolverrException("Unsupported POST Content-Type: " + request.Headers.ContentType);
                }
            }
            else
            {
                throw new FlareSolverrException("Unsupported HttpMethod: " + request.Method);
            }

            var apiUrl = string.Format("{0}/v1", Settings.Host.TrimEnd('/'));
            var newRequest = new HttpRequest(apiUrl, HttpAccept.Json);

            newRequest.Headers.ContentType = "application/json";
            newRequest.Method = HttpMethod.Post;
            newRequest.LogResponseContent = true;
            newRequest.RequestTimeout = TimeSpan.FromSeconds(Settings.RequestTimeout + 5);
            newRequest.SetContent(req.ToJson());
            newRequest.ContentSummary = req.ToJson(Formatting.None);

            _logger.Debug("Cloudflare Detected, Applying FlareSolverr Proxy {0} to request {1}", Name, request.Url);

            return newRequest;
        }

        public override ValidationResult Test()
        {
            var failures = new List<ValidationFailure>();

            var request = GenerateFlareSolverrRequest(_cloudRequestBuilder.Create()
                                  .Resource("/ping")
                                  .Build());

            try
            {
                var response = _httpClient.Execute(request);

                if (response.StatusCode != HttpStatusCode.OK)
                {
                    _logger.Error("Proxy validation failed: {0}", response.StatusCode);
                    failures.Add(new NzbDroneValidationFailure("Host", _localizationService.GetLocalizedString("ProxyValidationBadRequest", new Dictionary<string, object> { { "statusCode", response.StatusCode } })));
                }

                var result = JsonConvert.DeserializeObject<FlareSolverrResponse>(response.Content);
            }
            catch (Exception ex)
            {
                _logger.Error(ex, "Proxy validation failed");
                failures.Add(new NzbDroneValidationFailure("Host", _localizationService.GetLocalizedString("ProxyValidationUnableToConnect", new Dictionary<string, object> { { "exceptionMessage", ex.Message } })));
            }

            return new ValidationResult(failures);
        }

        private Uri GetProxyUri(HttpProxySettings proxySettings)
        {
            return proxySettings.Type switch
            {
                ProxyType.Http => new Uri("http://" + proxySettings.Host + ":" + proxySettings.Port),
                ProxyType.Socks4 => new Uri("socks4://" + proxySettings.Host + ":" + proxySettings.Port),
                ProxyType.Socks5 => new Uri("socks5://" + proxySettings.Host + ":" + proxySettings.Port),
                _ => null
            };
        }

        private class FlareSolverrRequest
        {
            public string Cmd { get; set; }
            public string Url { get; set; }
            public Cookie[] Cookies { get; set; }
            public FlareSolverrProxy Proxy { get; set; }
        }

        private class FlareSolverrRequestGet : FlareSolverrRequest
        {
            public string Headers { get; set; }
            public int MaxTimeout { get; set; }
        }

        private class FlareSolverrRequestPost : FlareSolverrRequest
        {
            public string PostData { get; set; }
            public int MaxTimeout { get; set; }
        }

        private class FlareSolverrRequestPostUrlEncoded : FlareSolverrRequestPost
        {
            public HeadersPost Headers { get; set; }
        }

        private class FlareSolverrProxy
        {
            public string Url { get; set; }
            public string Username { get; set; }
            public string Password { get; set; }
        }

        private class HeadersPost
        {
            public string ContentType { get; set; }
            public string ContentLength { get; set; }
        }

        private class FlareSolverrResponse
        {
            public string Status { get; set; }
            public string Message { get; set; }
            public long StartTimestamp { get; set; }
            public long EndTimestamp { get; set; }
            public string Version { get; set; }
            public Solution Solution { get; set; }
        }

        private class Solution
        {
            public string Url { get; set; }
            public string Status { get; set; }
            public Headers Headers { get; set; }
            public string Response { get; set; }
            public Cookie[] Cookies { get; set; }
            public string UserAgent { get; set; }
        }

        private class Cookie
        {
            public string Name { get; set; }
            public string Value { get; set; }
            public string Domain { get; set; }
            public string Path { get; set; }
            public double Expires { get; set; }
            public int Size { get; set; }
            public bool HttpOnly { get; set; }
            public bool Secure { get; set; }
            public bool Session { get; set; }
            public string SameSite { get; set; }

            public string ToHeaderValue() => $"{Name}={Value}";
            public System.Net.Cookie ToCookieObj() => new System.Net.Cookie(Name, Value);
        }

        private class Headers
        {
            public string Status { get; set; }
            public string Date { get; set; }

            [JsonProperty(PropertyName = "content-type")]
            public string ContentType { get; set; }
        }
    }
}