Add random delay option to same domain delay

README.md

@@ -169,6 +169,7 @@ Emitted when a task is queued via [Cluster.queue] or [Cluster.execute]. The firs
   - `retryLimit` <[number]> How often do you want to retry a job before marking it as failed. Ignored by tasks queued via [Cluster.execute]. Defaults to `0`.
   - `retryDelay` <[number]> How much time should pass at minimum between the job execution and its retry. Ignored by tasks queued via [Cluster.execute]. Defaults to `0`.
   - `sameDomainDelay` <[number]> How much time should pass at minimum between two requests to the same domain. If you use this field, the queued `data` must be your URL or `data` must be an object containing a field called `url`.
+  - `sameDomainRandomness` <[number]> How much time should be added/subtracted to the `sameDomainDelay`. It will pick a random number between `-sameDomainRandomness` and `+sameDomainRandomness`. This option is ignored if `sameDomainDelay` is not set or if its value is higher than `sameDomainDelay`.
   - `skipDuplicateUrls` <[boolean]> If set to `true`, will skip URLs which were already crawled by the cluster. Defaults to `false`. If you use this field, the queued `data` must be your URL or `data` must be an object containing a field called `url`.
   - `timeout` <[number]> Specify a timeout for all tasks. Defaults to `30000` (30 seconds).
   - `monitor` <[boolean]> If set to `true`, will provide a small command line output to provide information about the crawling process. Defaults to `false`.

src/Cluster.ts

@@ -27,6 +27,7 @@ interface ClusterOptions {
     retryDelay: number;
     skipDuplicateUrls: boolean;
     sameDomainDelay: number;
+    sameDomainRandomness: number;
     puppeteer: any;
 }
 
@@ -50,6 +51,7 @@ const DEFAULT_OPTIONS: ClusterOptions = {
     retryDelay: 0,
     skipDuplicateUrls: false,
     sameDomainDelay: 0,
+    sameDomainRandomness:0,
     puppeteer: undefined,
 };
 
@@ -100,6 +102,7 @@ export default class Cluster<JobData = any, ReturnData = any> extends EventEmitt
 
     private duplicateCheckUrls: Set<string> = new Set();
     private lastDomainAccesses: Map<string, number> = new Map();
+    private lastDomainRandomNumber: Map<string, number> = new Map();
 
     private systemMonitor: SystemMonitor = new SystemMonitor();
 
@@ -161,6 +164,9 @@ export default class Cluster<JobData = any, ReturnData = any> extends EventEmitt
         if (this.options.perBrowserOptions) {
             this.perBrowserOptions = [...this.options.perBrowserOptions];
         }
+        if (this.options.sameDomainDelay < this.options.sameDomainRandomness) {
+            this.options.sameDomainRandomness = 0;
+        }
 
         try {
             await this.browser.init();
@@ -283,10 +289,13 @@ export default class Cluster<JobData = any, ReturnData = any> extends EventEmitt
         // Check if the job needs to be delayed due to sameDomainDelay
         if (this.options.sameDomainDelay !== 0 && domain !== undefined) {
             const lastDomainAccess = this.lastDomainAccesses.get(domain);
+            const delay = this.options.sameDomainDelay +
+                ((this.lastDomainRandomNumber.get(domain) || 0) *
+                    this.options.sameDomainRandomness * 2) - this.options.sameDomainRandomness;
             if (lastDomainAccess !== undefined
-                && lastDomainAccess + this.options.sameDomainDelay > Date.now()) {
+                && lastDomainAccess + delay > Date.now()) {
                 this.jobQueue.push(job, {
-                    delayUntil: lastDomainAccess + this.options.sameDomainDelay,
+                    delayUntil: lastDomainAccess + delay,
                 });
                 this.work();
                 return;
@@ -298,6 +307,9 @@ export default class Cluster<JobData = any, ReturnData = any> extends EventEmitt
             this.duplicateCheckUrls.add(url);
         }
         if (this.options.sameDomainDelay !== 0 && domain !== undefined) {
+            if (this.options.sameDomainRandomness) {
+                this.lastDomainRandomNumber.set(domain, Math.random());
+            }
             this.lastDomainAccesses.set(domain, Date.now());
         }
 
@@ -370,7 +382,7 @@ export default class Cluster<JobData = any, ReturnData = any> extends EventEmitt
             // option: maxConcurrency
             (this.options.maxConcurrency === 0
                 || workerCount < this.options.maxConcurrency)
-            // just allow worker creaton every few milliseconds
+            // just allow worker creation every few milliseconds
             && (this.options.workerCreationDelay === 0
                 || this.lastLaunchedWorkerTime + this.options.workerCreationDelay < Date.now())
         );