Feature: Search limit (#418)

The idea here is to limit the number of queries to be made between each
running.

The current feature `excludeRecentSearch` can already filter off some
results, so combined with a limit feature we can have more flexibility.

Example of my usage:
- My library has 440 items
- `excludeRecentSearch` set to `5 days`
- `searchLimit` set to `100`
- `searchCadence` set to `1 day`

Every day the search runs, it picks only the 100 that have not be done
before

The user needs to pay attemption of their values or this can keep items
out of the search, for example, if the `excludeRecentSearch` is too
short for the limit, it will be start to pick items again...

fixes #432

src/cmd.ts

@@ -207,6 +207,12 @@ function createCommandWithSharedOptions(name, description) {
 			"--search-timeout <timeout>",
 			"Timeout for unresponsive searches",
 			fallback(fileConfig.searchTimeout, "30 seconds")
+		)
+		.option(
+			"--search-limit <number>",
+			"The number of searches before stops",
+			parseInt,
+			fallback(fileConfig.searchLimit, 0)
 		);
 }
 

src/config.template.cjs

@@ -215,4 +215,11 @@ module.exports = {
 	 * null
 	 */
 	searchTimeout: undefined,
+
+	/**
+	 * The number of searches to be done before it stop.
+	 * Combine this with "excludeRecentSearch" and "searchCadence" to smooth long-term API usage patterns.
+	 * Default is no limit.
+	 */
+	searchLimit: undefined,
 };

src/config.template.docker.cjs

@@ -219,4 +219,11 @@ module.exports = {
 	 * null
 	 */
 	searchTimeout: undefined,
+
+	/**
+	 * The number of searches to be done before stop.
+	 * Combine this with "excludeRecentSearch" and "searchCadence" to smooth long-term API usage patterns.
+	 * Default is no limit.
+	 */
+	searchLimit: undefined,
 };

src/configuration.ts

@@ -38,6 +38,7 @@ interface FileConfig {
 	rssCadence?: string;
 	snatchTimeout?: string;
 	searchTimeout?: string;
+	searchLimit?: number;
 }
 
 interface GenerateConfigParams {

src/pipeline.ts

@@ -229,7 +229,7 @@ export async function checkNewCandidateMatch(
 }
 
 async function findSearchableTorrents() {
-	const { torrents, dataDirs, torrentDir } = getRuntimeConfig();
+	const { torrents, dataDirs, torrentDir, searchLimit } = getRuntimeConfig();
 	let allSearchees: Searchee[] = [];
 	if (Array.isArray(torrents)) {
 		const searcheeResults = await Promise.all(
@@ -255,7 +255,7 @@ async function findSearchableTorrents() {
 	}
 
 	const hashesToExclude = allSearchees.map((t) => t.infoHash).filter(Boolean);
-	const filteredTorrents = await filterAsync(
+	let filteredTorrents = await filterAsync(
 		filterDupes(allSearchees).filter(filterByContent),
 		filterTimestamps
 	);
@@ -265,6 +265,15 @@ async function findSearchableTorrents() {
 		message: `Found ${allSearchees.length} torrents, ${filteredTorrents.length} suitable to search for matches`,
 	});
 
+	if (searchLimit && filteredTorrents.length > searchLimit) {
+		logger.info({
+			label: Label.SEARCH,
+			message: `Limited to ${searchLimit} searches`,
+		});
+
+		filteredTorrents = filteredTorrents.slice(0, searchLimit);
+	}
+
 	return { samples: filteredTorrents, hashesToExclude };
 }
 

src/runtimeConfig.ts

@@ -1,4 +1,5 @@
 import { Action, LinkType, MatchMode } from "./constants.js";
+
 export interface RuntimeConfig {
 	offset: number;
 	delay: number;
@@ -30,6 +31,7 @@ export interface RuntimeConfig {
 	rssCadence: number;
 	snatchTimeout: number;
 	searchTimeout: number;
+	searchLimit: number;
 }
 
 let runtimeConfig: RuntimeConfig;