Skip to content

Commit

Permalink
Fixed: (Anidub/Animedia) Use rate limit in sub-requests
Browse files Browse the repository at this point in the history
  • Loading branch information
@mynameisbogdan
mynameisbogdan committed Feb 7, 2023
1 parent f808204 commit 94c4554
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 128 deletions.
132 changes: 67 additions & 65 deletions src/NzbDrone.Core/Indexers/Definitions/Anidub.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ public Anidub(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, I

public override IIndexerRequestGenerator GetRequestGenerator()
{
return new AnidubRequestGenerator { Settings = Settings, Capabilities = Capabilities };
return new AnidubRequestGenerator(Settings);
}

public override IParseIndexerResponse GetParser()
{
return new AnidubParser(Settings, Capabilities.Categories) { HttpClient = _httpClient, Logger = _logger };
return new AnidubParser(Settings, Capabilities.Categories, RateLimit, _httpClient, _logger);
}

protected override async Task DoLogin()
Expand Down Expand Up @@ -135,21 +135,25 @@ private IndexerCapabilities SetCapabilities()

public class AnidubRequestGenerator : IIndexerRequestGenerator
{
public UserPassTorrentBaseSettings Settings { get; set; }
public IndexerCapabilities Capabilities { get; set; }
private readonly UserPassTorrentBaseSettings _settings;

public AnidubRequestGenerator(UserPassTorrentBaseSettings settings)
{
_settings = settings;
}

private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories)
private IEnumerable<IndexerRequest> GetPagedRequests(string term)
{
var requestUrl = string.Empty;
string requestUrl;
var isSearch = !string.IsNullOrWhiteSpace(term);

if (isSearch)
{
requestUrl = string.Format("{0}/index.php?do=search", Settings.BaseUrl.TrimEnd('/'));
requestUrl = $"{_settings.BaseUrl.TrimEnd('/')}/index.php?do=search";
}
else
{
requestUrl = Settings.BaseUrl;
requestUrl = _settings.BaseUrl;
}

var request = new IndexerRequest(requestUrl, HttpAccept.Html);
Expand Down Expand Up @@ -194,7 +198,7 @@ public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchC
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
}
Expand All @@ -203,7 +207,7 @@ public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCrit
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}"));

return pageableRequests;
}
Expand All @@ -212,7 +216,7 @@ public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchC
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
}
Expand All @@ -221,7 +225,7 @@ public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchC
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
}
Expand All @@ -230,7 +234,7 @@ public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCr
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
}
Expand All @@ -243,33 +247,37 @@ public class AnidubParser : IParseIndexerResponse
{
private readonly UserPassTorrentBaseSettings _settings;
private readonly IndexerCapabilitiesCategories _categories;
public IIndexerHttpClient HttpClient { get; set; }
public Logger Logger { get; set; }
private readonly TimeSpan _rateLimit;
private readonly IIndexerHttpClient _httpClient;
private readonly Logger _logger;

private static Dictionary<string, string> CategoriesMap => new Dictionary<string, string>
{
{ "/anime_tv/full", "14" },
{ "/anime_tv/anime_ongoing", "10" },
{ "/anime_tv/shonen", "11" },
{ "/anime_tv", "2" },
{ "/xxx", "13" },
{ "/manga", "15" },
{ "/ost", "16" },
{ "/podcast", "17" },
{ "/anime_movie", "3" },
{ "/anime_ova/anime_ona", "5" },
{ "/anime_ova", "4" },
{ "/dorama/japan_dorama", "6" },
{ "/dorama/korea_dorama", "7" },
{ "/dorama/china_dorama", "8" },
{ "/dorama", "9" },
{ "/anons_ongoing", "12" }
};

public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories)
private static Dictionary<string, string> CategoriesMap => new ()
{
{ "/anime_tv/full", "14" },
{ "/anime_tv/anime_ongoing", "10" },
{ "/anime_tv/shonen", "11" },
{ "/anime_tv", "2" },
{ "/xxx", "13" },
{ "/manga", "15" },
{ "/ost", "16" },
{ "/podcast", "17" },
{ "/anime_movie", "3" },
{ "/anime_ova/anime_ona", "5" },
{ "/anime_ova", "4" },
{ "/dorama/japan_dorama", "6" },
{ "/dorama/korea_dorama", "7" },
{ "/dorama/china_dorama", "8" },
{ "/dorama", "9" },
{ "/anons_ongoing", "12" }
};

public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories, TimeSpan rateLimit, IIndexerHttpClient httpClient, Logger logger)
{
_settings = settings;
_categories = categories;
_rateLimit = rateLimit;
_httpClient = httpClient;
_logger = logger;
}

private static string GetTitle(AngleSharp.Html.Dom.IHtmlDocument content, AngleSharp.Dom.IElement tabNode)
Expand Down Expand Up @@ -314,9 +322,9 @@ private static string GetQuality(AngleSharp.Dom.IElement releaseNode)

private static int GetReleaseLeechers(AngleSharp.Dom.IElement tabNode)
{
const string LeechersSelector = ".list.down > .li_swing_m";
const string leechersSelector = ".list.down > .li_swing_m";

var leechersStr = tabNode.QuerySelector(LeechersSelector).TextContent;
var leechersStr = tabNode.QuerySelector(leechersSelector).TextContent;
int.TryParse(leechersStr, out var leechers);
return leechers;
}
Expand All @@ -332,18 +340,18 @@ private static int GetReleaseSeeders(AngleSharp.Dom.IElement tabNode)

private static int GetReleaseGrabs(AngleSharp.Dom.IElement tabNode)
{
const string GrabsSelector = ".list.down > .li_download_m";
const string grabsSelector = ".list.down > .li_download_m";

var grabsStr = tabNode.QuerySelector(GrabsSelector).TextContent;
var grabsStr = tabNode.QuerySelector(grabsSelector).TextContent;
int.TryParse(grabsStr, out var grabs);
return grabs;
}

private static string GetDateFromDocument(AngleSharp.Html.Dom.IHtmlDocument content)
{
const string DateSelector = ".story_inf > li:nth-child(2)";
const string dateSelector = ".story_inf > li:nth-child(2)";

var domDate = content.QuerySelector(DateSelector).LastChild;
var domDate = content.QuerySelector(dateSelector).LastChild;

if (domDate?.NodeName != "#text")
{
Expand Down Expand Up @@ -384,16 +392,16 @@ private DateTime GetDateFromShowPage(AngleSharp.Html.Dom.IHtmlDocument content)
return utcDate.AddHours(-russianStandardTimeDiff);
}

Logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}");
_logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}");

return DateTime.UtcNow;
}

private static long GetReleaseSize(AngleSharp.Dom.IElement tabNode)
{
const string SizeSelector = ".list.down > .red";
const string sizeSelector = ".list.down > .red";

var sizeStr = tabNode.QuerySelector(SizeSelector).TextContent;
var sizeStr = tabNode.QuerySelector(sizeSelector).TextContent;
return ParseUtil.GetBytes(sizeStr);
}

Expand Down Expand Up @@ -433,11 +441,11 @@ private IList<TorrentInfo> ParseRelease(IndexerResponse indexerResponse)
var release = new TorrentInfo
{
Title = GetTitle(dom, t),
InfoUrl = indexerResponse.Request.Url.ToString(),
InfoUrl = indexerResponse.Request.Url.FullUri,
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1,

Guid = indexerResponse.Request.Url.ToString() + t.Id,
Guid = indexerResponse.Request.Url.FullUri + t.Id,
Seeders = GetReleaseSeeders(t),
Peers = GetReleaseSeeders(t) + GetReleaseLeechers(t),
Grabs = GetReleaseGrabs(t),
Expand All @@ -459,36 +467,30 @@ public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)

var parser = new HtmlParser();
var dom = parser.ParseDocument(indexerResponse.Content);
var domQuery = string.Empty;

if (indexerResponse.Request.Url.Query.Contains("do=search"))
{
domQuery = ".searchitem > h3 > a";
}
else
{
domQuery = "#dle-content > .story > .story_h > .lcol > h2 > a";
}

var links = dom.QuerySelectorAll(domQuery);
var links = dom.QuerySelectorAll(".searchitem > h3 > a[href], #dle-content > .story > .story_h > .lcol > h2 > a[href]");
foreach (var link in links)
{
var url = link.GetAttribute("href");

var releaseRequest = new IndexerRequest(url, HttpAccept.Html);
var releaseResponse = new IndexerResponse(releaseRequest, HttpClient.Execute(releaseRequest.HttpRequest));
var releaseRequest = new HttpRequestBuilder(url)
.WithRateLimit(_rateLimit.TotalSeconds)
.SetHeader("Referer", _settings.BaseUrl)
.Accept(HttpAccept.Html)
.Build();

var releaseIndexerRequest = new IndexerRequest(releaseRequest);
var releaseResponse = new IndexerResponse(releaseIndexerRequest, _httpClient.Execute(releaseIndexerRequest.HttpRequest));

// Throw common http errors here before we try to parse
if (releaseResponse.HttpResponse.HasHttpError)
{
if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests)
{
throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse);
}
else
{
throw new IndexerException(releaseResponse, "Http error code: " + releaseResponse.HttpResponse.StatusCode);
throw new TooManyRequestsException(releaseResponse.HttpRequest, releaseResponse.HttpResponse);
}

throw new IndexerException(releaseResponse, $"HTTP Error - {releaseResponse.HttpResponse.StatusCode}. {url}");
}

torrentInfos.AddRange(ParseRelease(releaseResponse));
Expand Down

0 comments on commit 94c4554

Please sign in to comment.