Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

New: (Cardigann) Paging Support - v9 #1479

Draft
wants to merge 5 commits into
base: develop
Choose a base branch
from
Draft

New: (Cardigann) Paging Support - v9 #1479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
712d95e
Cleanup request paging code
Qstick Feb 26, 2023
b42bf2c
New: (Cardigann) Paging Support
Qstick Feb 26, 2023
ec8025c
New: (Cardigann) Add AllowEmptyInputs
Qstick Feb 26, 2023
fb5b325
fixup! New: (Cardigann) Add AllowEmptyInputs
Qstick Feb 26, 2023
3ca6f83
fixup! New: (Cardigann) Paging Support
Qstick Feb 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListRequestGeneratorFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ public void should_use_categories_for_feed()
{
var results = Subject.GetSearchRequests(new MovieSearchCriteria { Categories = new[] { NewznabStandardCategory.MoviesSD.Id, NewznabStandardCategory.MoviesDVD.Id } });

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

page.Url.Query.Should().Contain("&category=1%2C2");
}
Expand All @@ -81,9 +81,9 @@ public void should_not_search_by_imdbid_if_not_supported()
_movieSearchCriteria.ImdbId = "0076759";
var results = Subject.GetSearchRequests(_movieSearchCriteria);

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

page.Url.Query.Should().Contain("type=imdb");
page.Url.Query.Should().Contain("query=tt0076759");
Expand All @@ -96,9 +96,9 @@ public void should_search_by_name_and_year_if_missing_imdbid()

var results = Subject.GetSearchRequests(_movieSearchCriteria);

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

page.Url.Query.Should().Contain("type=name");
page.Url.Query.Should().Contain("query=Star+Wars");
Expand Down
12 changes: 6 additions & 6 deletions src/NzbDrone.Core.Test/IndexerTests/HDBitsTests/HDBitsRequestGeneratorFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ public void should_search_by_imdbid_if_supported()
var results = Subject.GetSearchRequests(_movieSearchCriteria);
var imdbQuery = int.Parse(_movieSearchCriteria.ImdbId);

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

var encoding = HttpHeader.GetEncodingFromContentType(page.HttpRequest.Headers.ContentType);

Expand All @@ -95,9 +95,9 @@ public void should_search_by_tvdbid_season_episode_if_supported()
var results = Subject.GetSearchRequests(_tvSearchSeasonEpisodeCriteria);
var tvdbQuery = _tvSearchSeasonEpisodeCriteria.TvdbId;

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

var encoding = HttpHeader.GetEncodingFromContentType(page.HttpRequest.Headers.ContentType);

Expand All @@ -117,9 +117,9 @@ public void should_search_by_tvdbid_daily_episode_if_supported()
var results = Subject.GetSearchRequests(_tvSearchDailyEpisodeCriteria);
var tvdbQuery = _tvSearchDailyEpisodeCriteria.TvdbId;

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

var encoding = HttpHeader.GetEncodingFromContentType(page.HttpRequest.Headers.ContentType);

Expand Down
43 changes: 21 additions & 22 deletions src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ public void should_return_subsequent_pages()
_movieSearchCriteria.Offset = 0;
var results = Subject.GetSearchRequests(_movieSearchCriteria);

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var pages = results.GetAllTiers().First().Take(3).ToList();
var pages = results.Take(3).ToList();

pages[0].Url.FullUri.Should().Contain("&offset=0");
}
Expand All @@ -63,9 +63,9 @@ public void should_not_get_unlimited_pages()
{
var results = Subject.GetSearchRequests(_movieSearchCriteria);

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var pages = results.GetAllTiers().First().Take(500).ToList();
var pages = results.Take(500).ToList();

pages.Count.Should().BeLessThan(500);
}
Expand All @@ -77,9 +77,9 @@ public void should_not_search_by_imdbid_if_not_supported()

var results = Subject.GetSearchRequests(_movieSearchCriteria);

results.GetAllTiers().Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

page.Url.Query.Should().NotContain("imdbid=0076759");
page.Url.Query.Should().Contain("q=Star");
Expand All @@ -92,9 +92,9 @@ public void should_search_by_imdbid_if_supported()
_capabilities.MovieSearchParams = new List<MovieSearchParam> { MovieSearchParam.Q, MovieSearchParam.ImdbId };

var results = Subject.GetSearchRequests(_movieSearchCriteria);
results.GetTier(0).Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

page.Url.Query.Should().Contain("imdbid=0076759");
}
Expand All @@ -106,9 +106,9 @@ public void should_search_by_tmdbid_if_supported()
_capabilities.MovieSearchParams = new List<MovieSearchParam> { MovieSearchParam.Q, MovieSearchParam.TmdbId };

var results = Subject.GetSearchRequests(_movieSearchCriteria);
results.GetTier(0).Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

page.Url.Query.Should().Contain("tmdbid=11");
}
Expand All @@ -120,9 +120,9 @@ public void should_prefer_search_by_tmdbid_if_rid_supported()
_capabilities.MovieSearchParams = new List<MovieSearchParam> { MovieSearchParam.Q, MovieSearchParam.ImdbId, MovieSearchParam.TmdbId };

var results = Subject.GetSearchRequests(_movieSearchCriteria);
results.GetTier(0).Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetAllTiers().First().First();
var page = results.First();

page.Url.Query.Should().Contain("tmdbid=11");
page.Url.Query.Should().NotContain("imdbid=0076759");
Expand All @@ -136,9 +136,9 @@ public void should_use_aggregrated_id_search_if_supported()
_capabilities.MovieSearchParams = new List<MovieSearchParam> { MovieSearchParam.Q, MovieSearchParam.ImdbId, MovieSearchParam.TmdbId };

var results = Subject.GetSearchRequests(_movieSearchCriteria);
results.GetTier(0).Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetTier(0).First().First();
var page = results.First();

page.Url.Query.Should().Contain("tmdbid=11");
page.Url.Query.Should().Contain("imdbid=0076759");
Expand All @@ -150,10 +150,9 @@ public void should_not_use_aggregrated_id_search_if_no_ids_supported()
_capabilities.MovieSearchParams = new List<MovieSearchParam> { MovieSearchParam.Q };

var results = Subject.GetSearchRequests(_movieSearchCriteria);
results.Tiers.Should().Be(1);
results.GetTier(0).Should().HaveCount(1);
results.Should().HaveCount(1);

var page = results.GetTier(0).First().First();
var page = results.First();

page.Url.Query.Should().Contain("q=");
}
Expand All @@ -167,7 +166,7 @@ public void should_not_use_aggregrated_id_search_if_no_ids_are_known()

var results = Subject.GetSearchRequests(_movieSearchCriteria);

var page = results.GetTier(0).First().First();
var page = results.First();

page.Url.Query.Should().Contain("q=");
}
Expand All @@ -178,9 +177,9 @@ public void should_fallback_to_q()
_capabilities.MovieSearchParams = new List<MovieSearchParam> { MovieSearchParam.Q, MovieSearchParam.ImdbId, MovieSearchParam.TmdbId };

var results = Subject.GetSearchRequests(_movieSearchCriteria);
results.Tiers.Should().Be(1);
results.Should().HaveCount(1);

var pageTier2 = results.GetTier(0).First().First();
var pageTier2 = results.First();

pageTier2.Url.Query.Should().NotContain("tmdbid=11");
pageTier2.Url.Query.Should().NotContain("imdbid=0076759");
Expand All @@ -193,9 +192,9 @@ public void should_pad_seasons_for_tv_search()
_capabilities.TvSearchParams = new List<TvSearchParam> { TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep };

var results = Subject.GetSearchRequests(_tvSearchCriteria);
results.Tiers.Should().Be(1);
results.Should().HaveCount(1);

var pageTier = results.GetTier(0).First().First();
var pageTier = results.First();

pageTier.Url.Query.Should().Contain("season=00");
}
Expand Down
6 changes: 3 additions & 3 deletions src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ public abstract class SearchCriteriaBase
public string SearchTerm { get; set; }
public int[] Categories { get; set; }
public string SearchType { get; set; }
public int? Limit { get; set; }
public int? Offset { get; set; }
public int Limit { get; set; }
public int Offset { get; set; }
public string Source { get; set; }
public string Host { get; set; }

public override string ToString() => $"{SearchQuery}, Offset: {Offset ?? 0}, Limit: {Limit ?? 0}, Categories: [{string.Join(", ", Categories)}]";
public override string ToString() => $"{SearchQuery}, Offset: {Offset}, Limit: {Limit}, Categories: [{string.Join(", ", Categories)}]";

public virtual string SearchQuery => $"Term: [{SearchTerm}]";

Expand Down
4 changes: 2 additions & 2 deletions src/NzbDrone.Core/IndexerSearch/ReleaseSearchService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ private TSpec Get<TSpec>(NewznabRequest query, List<int> indexerIds, bool intera

spec.SearchTerm = query.q;
spec.SearchType = query.t;
spec.Limit = query.limit;
spec.Offset = query.offset;
spec.Limit = query.limit ?? 100;
spec.Offset = query.offset ?? 0;
spec.Source = query.source;
spec.Host = query.host;

Expand Down
2 changes: 1 addition & 1 deletion src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, I
/* Update Service will fall back if version # does not exist for an indexer per Ta */

private const string DEFINITION_BRANCH = "master";
private const int DEFINITION_VERSION = 8;
private const int DEFINITION_VERSION = 9;

// Used when moving yml to C#
private readonly List<string> _definitionBlocklist = new ()
Expand Down
36 changes: 10 additions & 26 deletions src/NzbDrone.Core/Indexers/Definitions/Anidex.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,45 +100,29 @@ public AnidexRequestGenerator(AnidexSettings settings, IndexerCapabilities capab
_capabilities = capabilities;
}

public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(MovieSearchCriteria searchCriteria)
{
return new IndexerPageableRequestChain();
return new List<IndexerRequest>();
}

public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(MusicSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories);
}

public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(TvSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories);
}

public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(BookSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories);
}

public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(BasicSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories);
}

private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories)
Expand Down
40 changes: 10 additions & 30 deletions src/NzbDrone.Core/Indexers/Definitions/Anidub.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,49 +194,29 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string term)
yield return request;
}

public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}");
}

public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(TvSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}"));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}");
}

public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(BasicSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}");
}

public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(MusicSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}");
}

public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(BookSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));

return pageableRequests;
return GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}");
}

public Func<IDictionary<string, string>> GetCookies { get; set; }
Expand Down
20 changes: 8 additions & 12 deletions src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,34 +107,30 @@ public AnimeBytesRequestGenerator()
{
}

public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(MovieSearchCriteria searchCriteria)
=> GetRequestWithSearchType(searchCriteria, "anime");

public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(MusicSearchCriteria searchCriteria)
=> GetRequestWithSearchType(searchCriteria, "music");

public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(TvSearchCriteria searchCriteria)
=> GetRequestWithSearchType(searchCriteria, "anime");

public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(BookSearchCriteria searchCriteria)
=> GetRequestWithSearchType(searchCriteria, "anime");

public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
public IEnumerable<IndexerRequest> GetSearchRequests(BasicSearchCriteria searchCriteria)
=> GetRequestWithSearchType(searchCriteria, "anime");

private IndexerPageableRequestChain GetRequestWithSearchType(SearchCriteriaBase searchCriteria, string searchType)
private IEnumerable<IndexerRequest> GetRequestWithSearchType(SearchCriteriaBase searchCriteria, string searchType)
{
var pageableRequests = new IndexerPageableRequestChain();

// TODO: Remove this once Prowlarr has proper support for non Pageable Indexers and can tell Sonarr that indexer doesn't support pagination in a proper way, for now just return empty release list on all request containing an offset
if (searchCriteria.Offset is > 0)
{
return pageableRequests;
return new List<IndexerRequest>();
}

pageableRequests.Add(GetRequest(searchType, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));

return pageableRequests;
return GetRequest(searchType, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories);
}

private IEnumerable<IndexerRequest> GetRequest(string searchType, string term, int[] categories)
Expand Down