fix(basic-crawler): limit internalTimeoutMillis in addition to `req…

…uestHandlerTimeoutMillis` (#1981) Closes #1766
apify · Jul 17, 2023 · 8122622 · 8122622
1 parent 4ba42fd
commit 8122622
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -532,11 +532,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         const maxSignedInteger = 2 ** 31 - 1;
         if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
             log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}`
-                + `does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
+                + ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
 
             this.requestHandlerTimeoutMillis = maxSignedInteger;
         }
 
+        this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
+
         let shouldLogMaxPagesExceeded = true;
         const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
 

diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
@@ -973,7 +973,7 @@ describe('BasicCrawler', () => {
         results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));
     });
 
-    test('limits handleRequestTimeoutSecs to a valid value', async () => {
+    test('limits handleRequestTimeoutSecs and derived vars to a valid value', async () => {
         const url = 'https://example.com';
         const requestList = await RequestList.open({ sources: [{ url }] });
 
@@ -991,6 +991,8 @@ describe('BasicCrawler', () => {
         const maxSignedInteger = 2 ** 31 - 1;
         // @ts-expect-error Accessing private prop
         expect(crawler.requestHandlerTimeoutMillis).toBe(maxSignedInteger);
+        // @ts-expect-error Accessing private prop
+        expect(crawler.internalTimeoutMillis).toBe(maxSignedInteger);
     });
 
     test('should not log stack trace for timeout errors by default', async () => {