fix: support DELETE requests in HttpCrawler (#2039)

This commit addresses an issue in the @crawlee/http (HttpCrawler) and
@crawlee/cheerio (CheerioCrawler) packages related to setting the DELETE
method for requests. The problem caused requests to fail with a timeout,
rendering the functionality unusable.

Closes #1658

packages/http-crawler/src/internals/http-crawler.ts

@@ -814,6 +814,11 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
                 }
             });
 
+            // We need to end the stream for DELETE requests, otherwise it will hang.
+            if (options.method && ['DELETE', 'delete'].includes(options.method)) {
+                stream.end();
+            }
+
             stream.on('error', reject);
             stream.on('response', () => {
                 resolve(addResponsePropertiesToStream(stream));

test/core/crawlers/cheerio_crawler.test.ts

@@ -507,7 +507,7 @@ describe('CheerioCrawler', () => {
         });
     });
 
-    test('should ignore non http error status codes set by user', async () => {
+    test('should ignore http error status codes set by user', async () => {
         const requestList = await getRequestListForMock({
             headers: {
                 'content-type': 'text/plain',
@@ -534,7 +534,7 @@ describe('CheerioCrawler', () => {
         expect(failed).toHaveLength(0);
     });
 
-    test('should throw and error on http error status codes set by user', async () => {
+    test('should throw an error on http error status codes set by user', async () => {
         const requestList = await getRequestListForMirror();
         const failed: Request[] = [];
 
@@ -1290,6 +1290,34 @@ describe('CheerioCrawler', () => {
             expect(cheerioCrawler.requestHandler).toBeUndefined();
         });
     });
+
+    test('should work with delete requests', async () => {
+        const sources: Source[] = [1, 2, 3, 4].map((num) => {
+            return {
+                url: `${serverAddress}/special/mock?a=${num}`,
+                method: 'DELETE',
+            };
+        });
+        const requestList = await RequestList.open(null, sources);
+
+        const failed: Request[] = [];
+
+        const cheerioCrawler = new CheerioCrawler({
+            requestList,
+            maxConcurrency: 1,
+            maxRequestRetries: 0,
+            navigationTimeoutSecs: 5,
+            requestHandlerTimeoutSecs: 5,
+            requestHandler: async () => {},
+            failedRequestHandler: async ({ request }) => {
+                failed.push(request);
+            },
+        });
+
+        await cheerioCrawler.run();
+
+        expect(failed).toHaveLength(0);
+    });
 });
 
 async function getRequestListForMock(mockData: Dictionary, pathName = 'special/mock') {

test/core/crawlers/http_crawler.test.ts

@@ -279,7 +279,7 @@ test('POST with undefined (empty) payload', async () => {
     expect(results).toStrictEqual(['']);
 });
 
-test('should ignore non http error status codes set by user', async () => {
+test('should ignore http error status codes set by user', async () => {
     const failed: any[] = [];
 
     const crawler = new HttpCrawler({
@@ -298,7 +298,7 @@ test('should ignore non http error status codes set by user', async () => {
     expect(failed).toHaveLength(0);
 });
 
-test('should throw and error on http error status codes set by user', async () => {
+test('should throw an error on http error status codes set by user', async () => {
     const failed: any[] = [];
 
     const crawler = new HttpCrawler({
@@ -316,3 +316,25 @@ test('should throw and error on http error status codes set by user', async () =
     expect(crawler.autoscaledPool.minConcurrency).toBe(2);
     expect(failed).toHaveLength(1);
 });
+
+test('should work with delete requests', async () => {
+    const failed: any[] = [];
+
+    const cheerioCrawler = new HttpCrawler({
+        maxConcurrency: 1,
+        maxRequestRetries: 0,
+        navigationTimeoutSecs: 5,
+        requestHandlerTimeoutSecs: 5,
+        requestHandler: async () => {},
+        failedRequestHandler: async ({ request }) => {
+            failed.push(request);
+        },
+    });
+
+    await cheerioCrawler.run([{
+        url: `${url}`,
+        method: 'DELETE',
+    }]);
+
+    expect(failed).toHaveLength(0);
+});