From f59cc953ec6493f536f142e08b428933f840e74c Mon Sep 17 00:00:00 2001 From: Bogdan Date: Mon, 1 May 2023 19:56:06 +0300 Subject: [PATCH 1/2] cardigann: check for page size --- .../Definitions/movietorrent.yml | 5 +++++ src/Jackett.Common/Definitions/schema.json | 15 ++++++++++++++ src/Jackett.Common/Indexers/BaseIndexer.cs | 1 + .../Indexers/CardigannIndexer.cs | 20 +++++++++++++++++++ .../Models/IndexerDefinition.cs | 4 ++++ 5 files changed, 45 insertions(+) diff --git a/src/Jackett.Common/Definitions/movietorrent.yml b/src/Jackett.Common/Definitions/movietorrent.yml index 179ee0228ca48..a96190b89ca99 100644 --- a/src/Jackett.Common/Definitions/movietorrent.yml +++ b/src/Jackett.Common/Definitions/movietorrent.yml @@ -10,6 +10,8 @@ links: - https://movietorrent.co/ caps: + limitsDefault: 36 + limitsMax: 36 categorymappings: - {id: 1, cat: Movies, desc: "Bollywood"} - {id: 2, cat: Movies/HD, desc: "1080p"} @@ -74,6 +76,9 @@ download: - name: validfilename search: + pageSize: 12 + pageable: true + paths: - path: "?s={{ .Keywords }}" - path: "/page/2/?s={{ .Keywords }}" diff --git a/src/Jackett.Common/Definitions/schema.json b/src/Jackett.Common/Definitions/schema.json index e5c1ac61015d9..f524f3ea1ae8f 100644 --- a/src/Jackett.Common/Definitions/schema.json +++ b/src/Jackett.Common/Definitions/schema.json @@ -93,6 +93,14 @@ "type": "object", "additionalProperties": false, "properties": { + "limitsDefault": { + "type": "integer", + "minimum": 1 + }, + "limitsMax": { + "type": "integer", + "minimum": 1 + }, "categories": { "type": "object", "additionalProperties": false, @@ -482,6 +490,13 @@ "type": "object", "additionalProperties": false, "properties": { + "pageSize": { + "type": "integer", + "minimum": 1 + }, + "pageable": { + "type": "boolean" + }, "path": { "type": "string" }, diff --git a/src/Jackett.Common/Indexers/BaseIndexer.cs b/src/Jackett.Common/Indexers/BaseIndexer.cs index f5afc3613f53d..af623270681d2 100644 --- a/src/Jackett.Common/Indexers/BaseIndexer.cs +++ b/src/Jackett.Common/Indexers/BaseIndexer.cs @@ -35,6 +35,7 @@ public abstract class BaseIndexer : IIndexer public virtual string Type { get; protected set; } public virtual bool SupportsPagination => false; + public virtual int PageSize => 0; public virtual bool IsConfigured { get; protected set; } public virtual string[] Tags { get; protected set; } diff --git a/src/Jackett.Common/Indexers/CardigannIndexer.cs b/src/Jackett.Common/Indexers/CardigannIndexer.cs index 4db00b9da7783..73c3ca707ca5a 100644 --- a/src/Jackett.Common/Indexers/CardigannIndexer.cs +++ b/src/Jackett.Common/Indexers/CardigannIndexer.cs @@ -27,6 +27,8 @@ namespace Jackett.Common.Indexers { public class CardigannIndexer : BaseWebIndexer { + public override int PageSize => Definition.Search != null && Definition.Search.PageSize > 0 ? Definition.Search.PageSize : 1; + protected IndexerDefinition Definition; protected WebResult landingResult; protected IHtmlDocument landingResultDocument; @@ -124,6 +126,8 @@ public CardigannIndexer(IIndexerConfigurationService configService, Utils.Client TorznabCaps = new TorznabCapabilities(); TorznabCaps.ParseCardigannSearchModes(Definition.Caps.Modes); TorznabCaps.SupportsRawSearch = Definition.Caps.Allowrawsearch; + TorznabCaps.LimitsDefault = Definition.Caps.LimitsDefault ?? TorznabCaps.LimitsDefault; + TorznabCaps.LimitsMax = Definition.Caps.LimitsMax ?? TorznabCaps.LimitsMax; // init config Data configData = new ConfigurationData(); @@ -1353,6 +1357,8 @@ protected override async Task> PerformQuery(TorznabQuer variables[".Query.Keywords"] = string.Join(" ", KeywordTokens); variables[".Keywords"] = applyFilters((string)variables[".Query.Keywords"], Search.Keywordsfilters, variables); + var pageSize = PageSize; + // TODO: prepare queries first and then send them parallel var SearchPaths = Search.Paths; foreach (var SearchPath in SearchPaths) @@ -1734,14 +1740,28 @@ protected override async Task> PerformQuery(TorznabQuer OnParseError(results, ex); } } + + pageSize = pageSize == 1 ? releases.Count : pageSize; + + if (Search.Pageable && !IsFullPage(releases, pageSize)) + { + break; + } } if (query.Limit > 0) + { releases = releases.Take(query.Limit).ToList(); + } return releases; } + protected virtual bool IsFullPage(IList page, int pageSize) + { + return pageSize != 0 && page.Count >= pageSize; + } + protected async Task handleRequest(requestBlock request, Dictionary variables = null, string referer = null) { var requestLinkStr = resolvePath(applyGoTemplateText(request.Path, variables)).ToString(); diff --git a/src/Jackett.Common/Models/IndexerDefinition.cs b/src/Jackett.Common/Models/IndexerDefinition.cs index 458f9051ac465..426251d01d417 100644 --- a/src/Jackett.Common/Models/IndexerDefinition.cs +++ b/src/Jackett.Common/Models/IndexerDefinition.cs @@ -69,6 +69,8 @@ public class CategorymappingBlock public class capabilitiesBlock { + public int? LimitsMax { get; set; } + public int? LimitsDefault { get; set; } public Dictionary Categories { get; set; } public List Categorymappings { get; set; } public Dictionary> Modes { get; set; } @@ -137,6 +139,8 @@ public class ratioBlock : selectorBlock public class searchBlock { + public int PageSize { get; set; } + public bool Pageable { get; set; } public string Path { get; set; } public List Paths { get; set; } public Dictionary> Headers { get; set; } From dc3e0680664478fb5ad2d802b078fc6a46a426ea Mon Sep 17 00:00:00 2001 From: Bogdan Date: Mon, 1 May 2023 21:10:16 +0300 Subject: [PATCH 2/2] cardigann: check for page size --- src/Jackett.Common/Definitions/1337x.yml | 5 +++++ src/Jackett.Common/Definitions/schema.json | 9 ++++++++- src/Jackett.Common/Indexers/CardigannIndexer.cs | 7 ++++++- src/Jackett.Common/Models/IndexerDefinition.cs | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/Jackett.Common/Definitions/1337x.yml b/src/Jackett.Common/Definitions/1337x.yml index e257b9d2dedb6..990bb15ad7a47 100644 --- a/src/Jackett.Common/Definitions/1337x.yml +++ b/src/Jackett.Common/Definitions/1337x.yml @@ -43,6 +43,8 @@ legacylinks: - https://1337x.unblockit.click/ caps: + limitsDefault: 80 + limitsMax: 80 categorymappings: # Anime - {id: 28, cat: TV/Anime, desc: "Anime/Anime"} @@ -181,6 +183,9 @@ download: attribute: href search: + pageSize: 20 + pageable: "{{ if or .Query.Album .Query.Artist .Keywords }}true{{ else }}false{{ end }}" + paths: # present first page of movies tv and music results if there are no search parms supplied (20 hits per page) - path: "{{ if or .Query.Album .Query.Artist .Keywords }}sort-search{{ else }}cat/Movies{{ end }}{{ if or .Query.Album .Query.Artist }}/{{ or .Query.Album .Query.Artist }}{{ else }}/{{ .Keywords }}{{ end }}{{ if or .Query.Album .Query.Artist .Keywords }}/{{ else }}{{ end }}{{ .Config.sort }}/{{ .Config.type }}/1/" diff --git a/src/Jackett.Common/Definitions/schema.json b/src/Jackett.Common/Definitions/schema.json index f524f3ea1ae8f..7a83cad3587bd 100644 --- a/src/Jackett.Common/Definitions/schema.json +++ b/src/Jackett.Common/Definitions/schema.json @@ -495,7 +495,14 @@ "minimum": 1 }, "pageable": { - "type": "boolean" + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ] }, "path": { "type": "string" diff --git a/src/Jackett.Common/Indexers/CardigannIndexer.cs b/src/Jackett.Common/Indexers/CardigannIndexer.cs index 73c3ca707ca5a..a626b6fa1eda1 100644 --- a/src/Jackett.Common/Indexers/CardigannIndexer.cs +++ b/src/Jackett.Common/Indexers/CardigannIndexer.cs @@ -1359,6 +1359,11 @@ protected override async Task> PerformQuery(TorznabQuer var pageSize = PageSize; + if (!bool.TryParse(applyGoTemplateText(Search.Pageable, variables), out var pageable)) + { + pageable = false; + } + // TODO: prepare queries first and then send them parallel var SearchPaths = Search.Paths; foreach (var SearchPath in SearchPaths) @@ -1743,7 +1748,7 @@ protected override async Task> PerformQuery(TorznabQuer pageSize = pageSize == 1 ? releases.Count : pageSize; - if (Search.Pageable && !IsFullPage(releases, pageSize)) + if (pageable && !IsFullPage(releases, pageSize)) { break; } diff --git a/src/Jackett.Common/Models/IndexerDefinition.cs b/src/Jackett.Common/Models/IndexerDefinition.cs index 426251d01d417..2a2ab7c1f416a 100644 --- a/src/Jackett.Common/Models/IndexerDefinition.cs +++ b/src/Jackett.Common/Models/IndexerDefinition.cs @@ -140,7 +140,7 @@ public class ratioBlock : selectorBlock public class searchBlock { public int PageSize { get; set; } - public bool Pageable { get; set; } + public string Pageable { get; set; } public string Path { get; set; } public List Paths { get; set; } public Dictionary> Headers { get; set; }