fix: enqueue link not finding relative links if the checked page is redirected

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -12,7 +12,7 @@ import {
     EVENT_SESSION_RETIRED,
     handleRequestTimeout,
     validators,
-    resolveBaseUrl,
+    resolveBaseUrlForEnqueueLinksFiltering,
     Configuration,
 } from '@crawlee/core';
 import type { BasicCrawlerOptions, Awaitable, Dictionary, RequestHandler, ErrorHandler } from '@crawlee/basic';
@@ -659,14 +659,14 @@ export async function browserCrawlerEnqueueLinks({
     originalRequestUrl,
     finalRequestUrl,
 }: EnqueueLinksInternalOptions) {
-    const baseUrl = resolveBaseUrl({
+    const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
         enqueueStrategy: options?.strategy,
         finalRequestUrl,
         originalRequestUrl,
         userProvidedBaseUrl: options?.baseUrl,
     });
 
-    const urls = await extractUrlsFromPage(page as any, options?.selector ?? 'a', baseUrl);
+    const urls = await extractUrlsFromPage(page as any, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
 
     return enqueueLinks({
         requestQueue,

packages/cheerio-crawler/src/internals/cheerio-crawler.ts

@@ -3,7 +3,7 @@ import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
 import type { BasicCrawlerOptions, ErrorHandler, RequestHandler } from '@crawlee/basic';
 import { BasicCrawler, BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '@crawlee/basic';
 import type { CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, Request, RequestQueue, Session } from '@crawlee/core';
-import { CrawlerExtension, enqueueLinks, mergeCookies, Router, resolveBaseUrl, validators } from '@crawlee/core';
+import { CrawlerExtension, enqueueLinks, mergeCookies, Router, resolveBaseUrlForEnqueueLinksFiltering, validators } from '@crawlee/core';
 import type { BatchAddRequestsResult, Awaitable, Dictionary } from '@crawlee/types';
 import type { CheerioRoot } from '@crawlee/utils';
 import { entries, parseContentTypeFromResponse } from '@crawlee/utils';
@@ -827,14 +827,14 @@ export async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, ori
         throw new Error('Cannot enqueue links because the DOM is not available.');
     }
 
-    const baseUrl = resolveBaseUrl({
+    const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
         enqueueStrategy: options?.strategy,
         finalRequestUrl,
         originalRequestUrl,
         userProvidedBaseUrl: options?.baseUrl,
     });
 
-    const urls = extractUrlsFromCheerio($, options?.selector ?? 'a', baseUrl);
+    const urls = extractUrlsFromCheerio($, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
 
     return enqueueLinks({
         requestQueue,

packages/core/src/enqueue_links/enqueue_links.ts

@@ -270,8 +270,12 @@ export async function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchA
 
 /**
  * @internal
+ * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
+ * - If a user provides a base url, we always return it
+ * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original request domain, or a redirected one
+ * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
  */
-export function resolveBaseUrl({
+export function resolveBaseUrlForEnqueueLinksFiltering({
     enqueueStrategy,
     finalRequestUrl,
     originalRequestUrl,
@@ -287,22 +291,11 @@ export function resolveBaseUrl({
 
     // We can assume users want to go off the domain in this case
     if (enqueueStrategy === EnqueueStrategy.All) {
-        return finalUrlOrigin ?? originalUrlOrigin;
+        return finalUrlOrigin;
     }
 
-    // Ensure links use the same hostname
-    if (enqueueStrategy === EnqueueStrategy.SameDomain) {
-        const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false })!;
-        const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false })!;
-
-        if (originalHostname === finalHostname) {
-            return finalUrlOrigin;
-        }
-
-        return undefined;
-    }
-
-    // Always enqueue urls that are from the same origin
+    // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
+    // before actually finding the urls
     return originalUrlOrigin;
 }