feat(Request): add requestHandlerTimeoutSecs and requestTimeoutSecs options

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -821,13 +821,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
 
         tryCancel();
 
+        // A request can have some request-specific timeouts defined. We check if they have been defined, and if so,
+        // use those timeouts instead. However; if they aren't present, the defaults will be used.
+        request!.requestTimeoutSecs ??= this.internalTimeoutMillis / 1e3;
+        request!.requestHandlerTimeoutSecs ??= this.requestHandlerTimeoutMillis / 1e3;
+
         if (this.useSessionPool) {
             await this._timeoutAndRetry(
                 async () => {
                     session = await this.sessionPool!.getSession();
                 },
-                this.internalTimeoutMillis,
-                `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
+                request!.requestTimeoutSecs * 1e3,
+                `Fetching session timed out after ${request!.requestTimeoutSecs} seconds.`,
             );
         }
 
@@ -888,14 +893,14 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         try {
             await addTimeoutToPromise(
                 () => this._runRequestHandler(crawlingContext),
-                this.requestHandlerTimeoutMillis,
-                `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${request.id}).`,
+                request.requestHandlerTimeoutSecs * 1e3,
+                `requestHandler timed out after ${request.requestHandlerTimeoutSecs} seconds (${request.id}).`,
             );
 
             await this._timeoutAndRetry(
                 () => source.markRequestHandled(request!),
-                this.internalTimeoutMillis,
-                `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
+                request.requestTimeoutSecs * 1e3,
+                `Marking request ${request.url} (${request.id}) as handled timed out after ${request.requestTimeoutSecs} seconds.`,
             );
 
             this.stats.finishJob(statisticsId);
@@ -907,8 +912,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             try {
                 await addTimeoutToPromise(
                     () => this._requestFunctionErrorHandler(err as Error, crawlingContext, source),
-                    this.internalTimeoutMillis,
-                    `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
+                    request.requestTimeoutSecs * 1e3,
+                    `Handling request failure of ${request.url} (${request.id}) timed out after ${request.requestTimeoutSecs} seconds.`,
                 );
             } catch (secondaryError: any) {
                 if (!secondaryError.triggeredFromUserHandler) {

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -11,6 +11,7 @@ import type {
     RequestHandler,
     ErrorHandler,
     EnqueueLinksOptions,
+    Request,
 } from '@crawlee/basic';
 import {
     cookieStringToToughCookie,
@@ -482,10 +483,12 @@ export abstract class BrowserCrawler<
             }
         }
 
+        request.requestHandlerTimeoutSecs ??= this.requestHandlerTimeoutMillis / 1e3;
+
         await addTimeoutToPromise(
             () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)),
-            this.requestHandlerTimeoutMillis,
-            `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`,
+            request.requestHandlerTimeoutSecs * 1e3,
+            `requestHandler timed out after ${request.requestHandlerTimeoutSecs} seconds.`,
         );
         tryCancel();
 
@@ -522,7 +525,9 @@ export abstract class BrowserCrawler<
     }
 
     protected async _handleNavigation(crawlingContext: Context) {
-        const gotoOptions = { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions;
+        crawlingContext.request.requestTimeoutSecs ??= this.navigationTimeoutMillis / 1e3;
+
+        const gotoOptions = { timeout: crawlingContext.request.requestTimeoutSecs * 1e3 } as unknown as GoToOptions;
 
         const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
 

packages/core/src/request.ts

@@ -28,6 +28,8 @@ const requestOptionalPredicates = {
     keepUrlFragment: ow.optional.boolean,
     useExtendedUniqueKey: ow.optional.boolean,
     skipNavigation: ow.optional.boolean,
+    requestHandlerTimeoutSecs: ow.optional.number,
+    requestTimeoutSecs: ow.optional.number,
 };
 
 /**
@@ -111,6 +113,16 @@ export class Request<UserData extends Dictionary = Dictionary> {
      */
     handledAt?: string;
 
+    /**
+     * The number of seconds to timeout after if the request hasn't succeeded yet.
+     */
+    requestTimeoutSecs?: number;
+
+    /**
+     * The number of seconds to timeout after if the `requestHandler` function runs for too long.
+     */
+    requestHandlerTimeoutSecs?: number;
+
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
      */
@@ -150,6 +162,8 @@ export class Request<UserData extends Dictionary = Dictionary> {
             keepUrlFragment = false,
             useExtendedUniqueKey = false,
             skipNavigation,
+            requestHandlerTimeoutSecs,
+            requestTimeoutSecs,
         } = options as RequestOptions & { loadedUrl?: string; retryCount?: number; errorMessages?: string[]; handledAt?: string | Date };
 
         let {
@@ -171,6 +185,8 @@ export class Request<UserData extends Dictionary = Dictionary> {
         this.errorMessages = [...errorMessages];
         this.headers = { ...headers };
         this.handledAt = handledAt as unknown instanceof Date ? (handledAt as Date).toISOString() : handledAt!;
+        this.requestHandlerTimeoutSecs = requestHandlerTimeoutSecs;
+        this.requestTimeoutSecs = requestTimeoutSecs;
 
         if (label) {
             userData.label = label;
@@ -399,12 +415,23 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      */
     skipNavigation?: boolean;
 
+    /**
+     * Determines a request-specific number of seconds before the request times out.
+     * Overrides the default `navigationTimeoutSecs` provided to the crawler.
+     */
+    requestTimeoutSecs?: number;
+
+    /**
+     * Determines a request-specific number of seconds before the request handler times out.
+     * Overrides the default `requestHandlerTimeoutSecs` provided to the crawler.
+     */
+    requestHandlerTimeoutSecs?: number;
+
     /** @internal */
     id?: string;
 
     /** @internal */
     handledAt?: string;
-
 }
 
 export interface PushErrorMessageOptions {

packages/http-crawler/src/internals/http-crawler.ts

@@ -452,10 +452,12 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
             });
         }
 
+        crawlingContext.request.requestTimeoutSecs ??= this.userRequestHandlerTimeoutMillis / 1e3;
+
         return addTimeoutToPromise(
             () => Promise.resolve(this.requestHandler(crawlingContext)),
-            this.userRequestHandlerTimeoutMillis,
-            `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`,
+            crawlingContext.request.requestTimeoutSecs * 1e3,
+            `requestHandler timed out after ${crawlingContext.request.requestTimeoutSecs} seconds.`,
         );
     }
 
@@ -475,10 +477,12 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
 
         const proxyUrl = crawlingContext.proxyInfo?.url;
 
+        crawlingContext.request.requestTimeoutSecs ??= this.navigationTimeoutMillis / 1e3;
+
         crawlingContext.response = await addTimeoutToPromise(
             () => this._requestFunction({ request, session, proxyUrl, gotOptions }),
-            this.navigationTimeoutMillis,
-            `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,
+            crawlingContext.request.requestTimeoutSecs * 1e3,
+            `request timed out after ${crawlingContext.request.requestTimeoutSecs} seconds.`,
         );
         tryCancel();
 

test/core/crawlers/basic_crawler.test.ts

@@ -1213,4 +1213,27 @@ describe('BasicCrawler', () => {
             expect(crawler).toBeTruthy();
         });
     });
+
+    it('Should use the requestHandlerTimeoutSecs provided within the request instead of the crawler one', async () => {
+        const timeStart = performance.now();
+
+        const crawler = new BasicCrawler({
+            requestHandlerTimeoutSecs: 99999,
+            maxRequestRetries: 1,
+            requestHandler: async () => {
+                await new Promise((resolve) => setTimeout(resolve, 5e4));
+            },
+            errorHandler: (_, error) => {
+                expect(error.message.includes('timed out after 2 seconds')).toBeTruthy();
+            },
+        });
+
+        await crawler.run([{
+            url: 'https://google.com',
+            requestHandlerTimeoutSecs: 2,
+        }]);
+
+        // The run should most definitely take less than 10 seconds with the 2 second timeout.
+        expect(performance.now() - timeStart).toBeLessThan(1e4);
+    });
 });

test/core/crawlers/cheerio_crawler.test.ts

@@ -1318,6 +1318,54 @@ describe('CheerioCrawler', () => {
             expect(cheerioCrawler.requestHandler).toBeUndefined();
         });
     });
+
+    test('Should use the requestHandlerTimeoutSecs provided within the request instead of the crawler one', async () => {
+        const timeStart = performance.now();
+
+        const crawler = new CheerioCrawler({
+            requestHandlerTimeoutSecs: 99999,
+            maxRequestRetries: 1,
+            requestHandler: async () => {
+                await new Promise((resolve) => setTimeout(resolve, 5e4));
+            },
+            errorHandler: (_, error) => {
+                expect(error.message.includes('timed out after 2 seconds')).toBeTruthy();
+            },
+        });
+
+        await crawler.run([{
+            url: 'https://google.com',
+            requestHandlerTimeoutSecs: 2,
+        }]);
+
+        // The run should most definitely take less than 10 seconds with the 2 second timeout.
+        expect(performance.now() - timeStart).toBeLessThan(1e4);
+    });
+
+    test('Should use the requestTimeoutSecs provided within the request instead of the crawler one', async () => {
+        const timeStart = performance.now();
+
+        const crawler = new CheerioCrawler({
+            navigationTimeoutSecs: 99999,
+            maxRequestRetries: 1,
+            requestHandler: async () => {
+                log.info('Hey');
+            },
+            errorHandler: (_, error) => {
+                expect(error.message.includes('timed out after 1 seconds')).toBeTruthy();
+            },
+        });
+
+        await crawler.run([{
+            // https://deelay.me/
+            // Google with a huge delay
+            url: 'https://deelay.me/200000/https://google.com',
+            requestTimeoutSecs: 1,
+        }]);
+
+        // The run should most definitely take less than 15 seconds with the 0.00001 second timeout.
+        expect(performance.now() - timeStart).toBeLessThan(15e3);
+    });
 });
 
 async function getRequestListForMock(port: number, mockData: Dictionary, pathName = 'mock') {