fix: add skipNavigation option to enqueueLinks (#2153)

Co-authored-by: Martin Adámek <banan23@gmail.com>
apify · Nov 2, 2023 · 118515d · 118515d
1 parent f188ebe
commit 118515d
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 3 deletions.
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -35,6 +35,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /** Sets {@apilink Request.label} for newly enqueued requests. */
     label?: string;
 
+    /**
+     * If set to `true`, tells the crawler to skip navigation and process the request directly.
+     * @default false
+     */
+    skipNavigation?: boolean;
+
     /**
      * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
      * since the relative URL resolution is done inside the browser automatically.
@@ -239,6 +245,7 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
         urls: ow.array.ofType(ow.string),
         requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
         forefront: ow.optional.boolean,
+        skipNavigation: ow.optional.boolean,
         limit: ow.optional.number,
         selector: ow.optional.string,
         baseUrl: ow.optional.string,

diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
@@ -208,7 +208,7 @@ export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatt
  */
 export function createRequestOptions(
     sources: (string | Record<string, unknown>)[],
-    options: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl'> = {},
+    options: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation'> = {},
 ): RequestOptions[] {
     return sources
         .map((src) => (typeof src === 'string' ? { url: src } : src as unknown as RequestOptions))
@@ -230,6 +230,10 @@ export function createRequestOptions(
                 };
             }
 
+            if (options.skipNavigation) {
+                requestOptions.skipNavigation = true;
+            }
+
             return requestOptions;
         });
 }

diff --git a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts
@@ -166,6 +166,12 @@ export interface EnqueueLinksByClickingElementsOptions {
      * @default false
      */
     forefront?: boolean;
+
+    /**
+     * If set to `true`, tells the crawler to skip navigation and process the request directly.
+     * @default false
+     */
+    skipNavigation?: boolean;
 }
 
 /**
@@ -233,6 +239,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClic
         maxWaitForPageIdleSecs: ow.optional.number,
         label: ow.optional.string,
         forefront: ow.optional.boolean,
+        skipNavigation: ow.optional.boolean,
     }));
 
     const {

diff --git a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts
@@ -167,6 +167,12 @@ export interface EnqueueLinksByClickingElementsOptions {
      * @default false
      */
     forefront?: boolean;
+
+    /**
+     * If set to `true`, tells the crawler to skip navigation and process the request directly.
+     * @default false
+     */
+    skipNavigation?: boolean;
 }
 
 /**
@@ -234,6 +240,7 @@ export async function enqueueLinksByClickingElements(options: EnqueueLinksByClic
         maxWaitForPageIdleSecs: ow.optional.number,
         label: ow.optional.string,
         forefront: ow.optional.boolean,
+        skipNavigation: ow.optional.boolean,
     }));
 
     const {

diff --git a/test/core/autoscaling/autoscaled_pool.test.ts b/test/core/autoscaling/autoscaled_pool.test.ts
@@ -541,5 +541,3 @@ describe('AutoscaledPool', () => {
         expect(Date.now() - now).toBeGreaterThanOrEqual(1e3);
     }, 10e3);
 });
-
-/* eslint-enable no-underscore-dangle */
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
@@ -189,6 +189,26 @@ describe('enqueueLinks()', () => {
             expect(enqueued[2].userData).toEqual({ label: 'COOL' });
         });
 
+        test('works with skipNavigation', async () => {
+            const { enqueued, requestQueue } = createRequestQueueMock();
+
+            await browserCrawlerEnqueueLinks({
+                options: {
+                    selector: '.click',
+                    skipNavigation: true,
+                },
+                page,
+                requestQueue,
+                originalRequestUrl: 'https://example.com',
+            });
+
+            expect(enqueued).toHaveLength(2);
+
+            for (const request of enqueued) {
+                expect(request.skipNavigation).toBe(true);
+            }
+        });
+
         test('works with exclude glob', async () => {
             const { enqueued, requestQueue } = createRequestQueueMock();
             const globs = [