feat: add support for sameDomainDelay

packages/basic-crawler/package.json

@@ -53,6 +53,7 @@
         "@crawlee/utils": "^3.4.2",
         "got-scraping": "^3.2.9",
         "ow": "^0.28.1",
+        "tldts": "^6.0.0",
         "tslib": "^2.4.0",
         "type-fest": "^4.0.0"
     }

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -46,6 +46,7 @@
 import type { Method, OptionsInit } from 'got-scraping';
 import { gotScraping } from 'got-scraping';
 import ow, { ArgumentError } from 'ow';
+import { getDomain } from 'tldts';
 import type { SetRequired } from 'type-fest';
 
 export interface BasicCrawlingContext<
@@ -220,6 +221,12 @@
     maxRequestRetries?: number;
 
     /**
+     * Indicates how much time wait before crawling same domain request
+     * @default 0
+     */
+    sameDomainDelay?: number;
+
+    /** 
      * Maximum number of session rotations per request.
      * The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
      *
@@ -434,6 +441,8 @@
     protected requestHandlerTimeoutMillis!: number;
     protected internalTimeoutMillis: number;
     protected maxRequestRetries: number;
+    protected sameDomainDelay: number;
+    protected domainAccessedTime: Map<string, number>;
     protected maxSessionRotations: number;
     protected handledRequestsCount: number;
     protected statusMessageLoggingInterval: number;
@@ -463,6 +472,7 @@
         // TODO: remove in a future release
         handleFailedRequestFunction: ow.optional.function,
         maxRequestRetries: ow.optional.number,
+        sameDomainDelay: ow.optional.number,
         maxSessionRotations: ow.optional.number,
         maxRequestsPerCrawl: ow.optional.number,
         autoscaledPoolOptions: ow.optional.object,
@@ -494,6 +504,7 @@
             requestList,
             requestQueue,
             maxRequestRetries = 3,
+            sameDomainDelay = 0,
             maxSessionRotations = 10,
             maxRequestsPerCrawl,
             autoscaledPoolOptions = {},
@@ -533,6 +544,7 @@
         this.statusMessageLoggingInterval = statusMessageLoggingInterval;
         this.statusMessageCallback = statusMessageCallback as StatusMessageCallback;
         this.events = config.getEventManager();
+        this.domainAccessedTime = new Map();
 
         this._handlePropertyNameChange({
             newName: 'requestHandler',
@@ -589,6 +601,7 @@
         }
 
         this.maxRequestRetries = maxRequestRetries;
+        this.sameDomainDelay = sameDomainDelay;
         this.maxSessionRotations = maxSessionRotations;
         this.handledRequestsCount = 0;
         this.stats = new Statistics({ logMessage: `${log.getOptions().prefix} request statistics:`, config });
@@ -964,6 +977,31 @@
         _crawlingContext: Context,
     ) {}
 
+    protected _handleRequestWithDelay(request:Request, source: RequestQueue | RequestList) {
+        const domain = getDomain(request.url);
+
+        if (!domain || !request) {
+            return false;
+        }
+
+        const now = Date.now();
+        const lastAccessTime = this.domainAccessedTime.get(domain);
+
+        if (!lastAccessTime || (now - lastAccessTime) >= this.sameDomainDelay) {
+            this.domainAccessedTime.set(domain, now);
+            return false;
+        }
+
+        const delay = lastAccessTime + this.sameDomainDelay - now;
+        this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
+        setTimeout(async () => {
+            this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
+            await source?.reclaimRequest(request);
+        }, delay);
+
+        return true;
+    }
+
     /**
      * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
      * then retries them in a case of an error, etc.
@@ -996,7 +1034,9 @@
 
         tryCancel();
 
-        if (!request) return;
+        if (!request || this._handleRequestWithDelay(request, source)) {
+            return;
+        }
 
         // Reset loadedUrl so an old one is not carried over to retries.
         request.loadedUrl = undefined;

yarn.lock

@@ -750,6 +750,7 @@ __metadata:
     "@crawlee/utils": ^3.4.2
     got-scraping: ^3.2.9
     ow: ^0.28.1
+    tldts: ^6.0.0
     tslib: ^2.4.0
     type-fest: ^4.0.0
   languageName: unknown