From 88c85f797ecab4294c301dc5a13b12067d2332d8 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Fri, 22 Mar 2024 10:34:06 +0200 Subject: [PATCH 01/12] feat: make RequestQueueV2 the default queue --- docs/experiments/request_locking.mdx | 14 +++++++++ .../src/internals/basic-crawler.ts | 29 ++++++------------- packages/core/src/storages/index.ts | 8 +++-- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/docs/experiments/request_locking.mdx b/docs/experiments/request_locking.mdx index f4f00116fff..e0c5e4b6380 100644 --- a/docs/experiments/request_locking.mdx +++ b/docs/experiments/request_locking.mdx @@ -6,6 +6,20 @@ description: Parallelize crawlers with ease using request locking import ApiLink from '@site/src/components/ApiLink'; +:::tip Release announcement + +As of **March 2024** (`crawlee` version `3.9.0`), this experiment is now enabled by default! With that said, if you encounter issues you can: + +- set `disableRequestLocking` to `true` in the `experiments` object of your crawler options +- update all imports of `RequestQueue` to `RequestQueueV1` +- open an issue on our [GitHub repository](https://github.com/apify/crawlee) + +The content below is kept for documentation purposes. + +::: + +--- + :::caution This is an experimental feature. While we welcome testers, keep in mind that it is currently not recommended to use this in production. diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 2a00b012af4..20bcf2e5ed1 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -356,10 +356,10 @@ export interface BasicCrawlerOptions Date: Fri, 22 Mar 2024 10:41:22 +0200 Subject: [PATCH 02/12] chore: WOOPSIE DAISY --- test/core/storages/request_queue.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.ts index 8ad854d9e11..8eb3b3527e8 100644 --- a/test/core/storages/request_queue.test.ts +++ b/test/core/storages/request_queue.test.ts @@ -4,7 +4,7 @@ import { QUERY_HEAD_MIN_LENGTH, API_PROCESSED_REQUESTS_DELAY_MILLIS, STORAGE_CONSISTENCY_DELAY_MILLIS, - RequestQueue, + RequestQueueV1 as RequestQueue, Request, Configuration, ProxyConfiguration, From c3b907061700f1dcc9c22f28094172f1dfb504a4 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Fri, 22 Mar 2024 10:52:10 +0200 Subject: [PATCH 03/12] chore: deprecate V2 export --- packages/core/src/storages/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.ts index 2bcaad2cda1..624229025d6 100644 --- a/packages/core/src/storages/index.ts +++ b/packages/core/src/storages/index.ts @@ -4,6 +4,7 @@ export * from './request_list'; export * from './request_provider'; export { RequestQueue as RequestQueueV1 } from './request_queue'; export { + /** @deprecated Import `RequestQueue` instead */ RequestQueueV2, // Export this as RequestQueue to avoid breaking changes (and to push it to default) RequestQueueV2 as RequestQueue, From 53209f898a1e828d1bb6f86a8f7279df32597316 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Fri, 22 Mar 2024 11:50:46 +0200 Subject: [PATCH 04/12] chore: bring back the old field but make it exist in types only --- packages/basic-crawler/src/internals/basic-crawler.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 20bcf2e5ed1..a87dfb025c2 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -360,6 +360,14 @@ export interface CrawlerExperiments { * default causes issues. Please open an issue if you encounter problems with the new API. */ disableRequestLocking?: boolean; + + /** + * @deprecated This experiment is now enabled by default, and this flag will be removed in a future release. + * If you encounter issues due to this change, please: + * - report it to us: https://github.com/apify/crawlee + * - set `disableRequestLocking` to `true` in the `experiments` option of the crawler + */ + requestLocking?: boolean; } /** From af21cc33bf94a5bf169f3d56f1fabbe3f1f3e382 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Mon, 25 Mar 2024 11:12:43 +0200 Subject: [PATCH 05/12] chore: maybe this fixes docs? --- .../core/src/storages/request_provider.ts | 17 +++++ packages/core/src/storages/request_queue.ts | 22 ++++++ .../core/src/storages/request_queue_v2.ts | 69 +++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/packages/core/src/storages/request_provider.ts b/packages/core/src/storages/request_provider.ts index 6fd85ca9161..97afe2b3948 100644 --- a/packages/core/src/storages/request_provider.ts +++ b/packages/core/src/storages/request_provider.ts @@ -375,6 +375,23 @@ export abstract class RequestProvider implements IStorage { return new Request(requestOptions as unknown as RequestOptions); } + /** + * Returns a next request in the queue to be processed, or `null` if there are no more pending requests. + * + * Once you successfully finish processing of the request, you need to call + * {@apilink RequestQueue.markRequestHandled} + * to mark the request as handled in the queue. If there was some error in processing the request, + * call {@apilink RequestQueue.reclaimRequest} instead, + * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function. + * + * Note that the `null` return value doesn't mean the queue processing finished, + * it means there are currently no pending requests. + * To check whether all requests in queue were finished, + * use {@apilink RequestQueue.isFinished} instead. + * + * @returns + * Returns the request object or `null` if there are no more pending requests. + */ abstract fetchNextRequest(options?: RequestOptions): Promise | null>; /** diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts index ae8ed86a81f..a282df11056 100644 --- a/packages/core/src/storages/request_queue.ts +++ b/packages/core/src/storages/request_queue.ts @@ -327,6 +327,12 @@ export class RequestQueue extends RequestProvider { return super.markRequestHandled(...args); } + /** + * Reclaims a failed request back to the queue, so that it can be returned for processing later again + * by another call to {@apilink RequestQueue.fetchNextRequest}. + * The request record in the queue is updated using the provided `request` parameter. + * For example, this lets you store the number of retries or error messages for the request. + */ override async reclaimRequest(...args: Parameters) { checkStorageAccess(); @@ -359,6 +365,22 @@ export class RequestQueue extends RequestProvider { this.lastActivity = new Date(); } + /** + * Opens a request queue and returns a promise resolving to an instance + * of the {@apilink RequestQueue} class. + * + * {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud. + * The queue is used for deep crawling of websites, where you start with several URLs and then + * recursively follow links to other pages. The data structure supports both breadth-first + * and depth-first crawling orders. + * + * For more details and code examples, see the {@apilink RequestQueue} class. + * + * @param [queueIdOrName] + * ID or name of the request queue to be opened. If `null` or `undefined`, + * the function returns the default request queue associated with the crawler run. + * @param [options] Open Request Queue options. + */ static override async open(...args: Parameters): Promise { return super.open(...args) as Promise; } diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.ts index 58af926b265..f82c9aa748a 100644 --- a/packages/core/src/storages/request_queue_v2.ts +++ b/packages/core/src/storages/request_queue_v2.ts @@ -21,6 +21,53 @@ const MAX_CACHED_REQUESTS = 2_000_000; */ const RECENTLY_HANDLED_CACHE_SIZE = 1000; +/** + * Represents a queue of URLs to crawl, which is used for deep crawling of websites + * where you start with several URLs and then recursively + * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders. + * + * Each URL is represented using an instance of the {@apilink Request} class. + * The queue can only contain unique URLs. More precisely, it can only contain {@apilink Request} instances + * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden. + * To add a single URL multiple times to the queue, + * corresponding {@apilink Request} objects will need to have different `uniqueKey` properties. + * + * Do not instantiate this class directly, use the {@apilink RequestQueue.open} function instead. + * + * `RequestQueue` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler} + * and {@apilink PlaywrightCrawler} as a source of URLs to crawl. + * Unlike {@apilink RequestList}, `RequestQueue` supports dynamic adding and removing of requests. + * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch. + * + * `RequestQueue` stores its data either on local disk or in the Apify Cloud, + * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set. + * + * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in + * that directory in an SQLite database file. + * + * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the + * [Apify Request Queue](https://docs.apify.com/storage/request-queue) + * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud` + * option to {@apilink RequestQueue.open} function, + * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set. + * + * **Example usage:** + * + * ```javascript + * // Open the default request queue associated with the crawler run + * const queue = await RequestQueue.open(); + * + * // Open a named request queue + * const queueWithName = await RequestQueue.open('some-name'); + * + * // Enqueue few requests + * await queue.addRequest({ url: 'http://example.com/aaa' }); + * await queue.addRequest({ url: 'http://example.com/bbb' }); + * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true }); + * ``` + * @category Sources + */ + class RequestQueue extends RequestProvider { private _listHeadAndLockPromise: Promise | null = null; @@ -143,6 +190,12 @@ class RequestQueue extends RequestProvider { return request; } + /** + * Reclaims a failed request back to the queue, so that it can be returned for processing later again + * by another call to {@apilink RequestQueue.fetchNextRequest}. + * The request record in the queue is updated using the provided `request` parameter. + * For example, this lets you store the number of retries or error messages for the request. + */ override async reclaimRequest(...args: Parameters): ReturnType { checkStorageAccess(); @@ -350,6 +403,22 @@ class RequestQueue extends RequestProvider { } } + /** + * Opens a request queue and returns a promise resolving to an instance + * of the {@apilink RequestQueue} class. + * + * {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud. + * The queue is used for deep crawling of websites, where you start with several URLs and then + * recursively follow links to other pages. The data structure supports both breadth-first + * and depth-first crawling orders. + * + * For more details and code examples, see the {@apilink RequestQueue} class. + * + * @param [queueIdOrName] + * ID or name of the request queue to be opened. If `null` or `undefined`, + * the function returns the default request queue associated with the crawler run. + * @param [options] Open Request Queue options. + */ static override async open(...args: Parameters): Promise { return super.open(...args) as Promise; } From a6b65fb1eb89e6d363b75cbc4a4dcf3ec262bc8e Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Mon, 25 Mar 2024 11:22:09 +0200 Subject: [PATCH 06/12] chore: will this work? --- packages/core/src/storages/index.ts | 4 ++-- packages/core/src/storages/request_queue_v2.ts | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.ts index 624229025d6..be2e3ba1f50 100644 --- a/packages/core/src/storages/index.ts +++ b/packages/core/src/storages/index.ts @@ -5,9 +5,9 @@ export * from './request_provider'; export { RequestQueue as RequestQueueV1 } from './request_queue'; export { /** @deprecated Import `RequestQueue` instead */ - RequestQueueV2, + RequestQueue as RequestQueueV2, // Export this as RequestQueue to avoid breaking changes (and to push it to default) - RequestQueueV2 as RequestQueue, + RequestQueue, } from './request_queue_v2'; export * from './storage_manager'; export * from './utils'; diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.ts index f82c9aa748a..4285078b7c6 100644 --- a/packages/core/src/storages/request_queue_v2.ts +++ b/packages/core/src/storages/request_queue_v2.ts @@ -68,7 +68,7 @@ const RECENTLY_HANDLED_CACHE_SIZE = 1000; * @category Sources */ -class RequestQueue extends RequestProvider { +export class RequestQueue extends RequestProvider { private _listHeadAndLockPromise: Promise | null = null; constructor(options: RequestProviderOptions, config = Configuration.getGlobalConfig()) { @@ -423,5 +423,3 @@ class RequestQueue extends RequestProvider { return super.open(...args) as Promise; } } - -export { RequestQueue as RequestQueueV2 }; From 3e68aeb970813a87ae3511a9e779d092407ebcf8 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Tue, 26 Mar 2024 10:42:43 +0200 Subject: [PATCH 07/12] chore: fix docs --- packages/core/src/storages/index.ts | 10 +++------- packages/core/src/storages/request_queue.ts | 8 +++++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/core/src/storages/index.ts b/packages/core/src/storages/index.ts index be2e3ba1f50..fd8260f7324 100644 --- a/packages/core/src/storages/index.ts +++ b/packages/core/src/storages/index.ts @@ -2,13 +2,9 @@ export * from './dataset'; export * from './key_value_store'; export * from './request_list'; export * from './request_provider'; -export { RequestQueue as RequestQueueV1 } from './request_queue'; -export { - /** @deprecated Import `RequestQueue` instead */ - RequestQueue as RequestQueueV2, - // Export this as RequestQueue to avoid breaking changes (and to push it to default) - RequestQueue, -} from './request_queue_v2'; +export { RequestQueueV1 } from './request_queue'; +export { RequestQueue } from './request_queue_v2'; +export { RequestQueue as RequestQueueV2 } from './request_queue_v2'; export * from './storage_manager'; export * from './utils'; export * from './access_checking'; diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts index a282df11056..3f33340eaa1 100644 --- a/packages/core/src/storages/request_queue.ts +++ b/packages/core/src/storages/request_queue.ts @@ -72,8 +72,10 @@ const RECENTLY_HANDLED_CACHE_SIZE = 1000; * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true }); * ``` * @category Sources + * + * @deprecated RequestQueue v1 is deprecated and will be removed in the future. Please use {@apilink RequestQueue} instead. */ -export class RequestQueue extends RequestProvider { +export class RequestQueueV1 extends RequestProvider { private queryQueueHeadPromise?: Promise<{ wasLimitReached: boolean; prevLimit: number; @@ -381,7 +383,7 @@ export class RequestQueue extends RequestProvider { * the function returns the default request queue associated with the crawler run. * @param [options] Open Request Queue options. */ - static override async open(...args: Parameters): Promise { - return super.open(...args) as Promise; + static override async open(...args: Parameters): Promise { + return super.open(...args) as Promise; } } From f047c8cfdb23b56111a037ad2e536c40af99f665 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Tue, 26 Mar 2024 10:45:15 +0200 Subject: [PATCH 08/12] fix: actually use the right rq version Woops. --- packages/basic-crawler/src/internals/basic-crawler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index a87dfb025c2..603e1a69a16 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -39,8 +39,8 @@ import { mergeCookies, NonRetryableError, purgeDefaultStorages, + RequestQueueV1, RequestQueue, - RequestQueueV2, RequestState, RetryRequestError, Router, @@ -1573,10 +1573,10 @@ export class BasicCrawler Date: Tue, 26 Mar 2024 11:12:35 +0200 Subject: [PATCH 09/12] fix: my bad --- packages/core/src/storages/request_queue.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts index 3f33340eaa1..bc9d16a6fb1 100644 --- a/packages/core/src/storages/request_queue.ts +++ b/packages/core/src/storages/request_queue.ts @@ -75,7 +75,7 @@ const RECENTLY_HANDLED_CACHE_SIZE = 1000; * * @deprecated RequestQueue v1 is deprecated and will be removed in the future. Please use {@apilink RequestQueue} instead. */ -export class RequestQueueV1 extends RequestProvider { +class RequestQueue extends RequestProvider { private queryQueueHeadPromise?: Promise<{ wasLimitReached: boolean; prevLimit: number; @@ -383,7 +383,9 @@ export class RequestQueueV1 extends RequestProvider { * the function returns the default request queue associated with the crawler run. * @param [options] Open Request Queue options. */ - static override async open(...args: Parameters): Promise { - return super.open(...args) as Promise; + static override async open(...args: Parameters): Promise { + return super.open(...args) as Promise; } } + +export { RequestQueue as RequestQueueV1 }; From 36394b3b7a46a60aeb2c83f08bdef4e59ec3a7bb Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Tue, 26 Mar 2024 21:01:14 +0200 Subject: [PATCH 10/12] chore: requested changes --- docs/experiments/request_locking.mdx | 4 +- .../src/internals/basic-crawler.ts | 15 ++---- .../core/src/storages/request_queue_v2.ts | 49 ++----------------- 3 files changed, 10 insertions(+), 58 deletions(-) diff --git a/docs/experiments/request_locking.mdx b/docs/experiments/request_locking.mdx index e0c5e4b6380..927c268ad8c 100644 --- a/docs/experiments/request_locking.mdx +++ b/docs/experiments/request_locking.mdx @@ -8,9 +8,9 @@ import ApiLink from '@site/src/components/ApiLink'; :::tip Release announcement -As of **March 2024** (`crawlee` version `3.9.0`), this experiment is now enabled by default! With that said, if you encounter issues you can: +As of **April 2024** (`crawlee` version `3.9.0`), this experiment is now enabled by default! With that said, if you encounter issues you can: -- set `disableRequestLocking` to `true` in the `experiments` object of your crawler options +- set `requestLocking` to `false` in the `experiments` object of your crawler options - update all imports of `RequestQueue` to `RequestQueueV1` - open an issue on our [GitHub repository](https://github.com/apify/crawlee) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 603e1a69a16..1b3ff9ddafe 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -355,17 +355,11 @@ export interface BasicCrawlerOptions | null = null; @@ -110,21 +97,7 @@ export class RequestQueue extends RequestProvider { } /** - * Returns a next request in the queue to be processed, or `null` if there are no more pending requests. - * - * Once you successfully finish processing of the request, you need to call - * {@apilink RequestQueue.markRequestHandled} - * to mark the request as handled in the queue. If there was some error in processing the request, - * call {@apilink RequestQueue.reclaimRequest} instead, - * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function. - * - * Note that the `null` return value doesn't mean the queue processing finished, - * it means there are currently no pending requests. - * To check whether all requests in queue were finished, - * use {@apilink RequestQueue.isFinished} instead. - * - * @returns - * Returns the request object or `null` if there are no more pending requests. + * @inheritDoc */ override async fetchNextRequest(): Promise | null> { checkStorageAccess(); @@ -191,10 +164,7 @@ export class RequestQueue extends RequestProvider { } /** - * Reclaims a failed request back to the queue, so that it can be returned for processing later again - * by another call to {@apilink RequestQueue.fetchNextRequest}. - * The request record in the queue is updated using the provided `request` parameter. - * For example, this lets you store the number of retries or error messages for the request. + * @inheritDoc */ override async reclaimRequest(...args: Parameters): ReturnType { checkStorageAccess(); @@ -404,20 +374,7 @@ export class RequestQueue extends RequestProvider { } /** - * Opens a request queue and returns a promise resolving to an instance - * of the {@apilink RequestQueue} class. - * - * {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud. - * The queue is used for deep crawling of websites, where you start with several URLs and then - * recursively follow links to other pages. The data structure supports both breadth-first - * and depth-first crawling orders. - * - * For more details and code examples, see the {@apilink RequestQueue} class. - * - * @param [queueIdOrName] - * ID or name of the request queue to be opened. If `null` or `undefined`, - * the function returns the default request queue associated with the crawler run. - * @param [options] Open Request Queue options. + * @inheritDoc */ static override async open(...args: Parameters): Promise { return super.open(...args) as Promise; From 0c8dd95c5ce1e5df607353aeaaa891313535f698 Mon Sep 17 00:00:00 2001 From: drobnikj Date: Mon, 13 May 2024 16:12:51 +0200 Subject: [PATCH 11/12] fix: update release date and adding blogpost about new changes --- docs/experiments/request_locking.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/experiments/request_locking.mdx b/docs/experiments/request_locking.mdx index 927c268ad8c..b1608e5d9ca 100644 --- a/docs/experiments/request_locking.mdx +++ b/docs/experiments/request_locking.mdx @@ -8,13 +8,14 @@ import ApiLink from '@site/src/components/ApiLink'; :::tip Release announcement -As of **April 2024** (`crawlee` version `3.9.0`), this experiment is now enabled by default! With that said, if you encounter issues you can: +As of **May 2024** (`crawlee` version `3.9.0`), this experiment is now enabled by default! With that said, if you encounter issues you can: - set `requestLocking` to `false` in the `experiments` object of your crawler options - update all imports of `RequestQueue` to `RequestQueueV1` - open an issue on our [GitHub repository](https://github.com/apify/crawlee) The content below is kept for documentation purposes. +If you're interested in the changes, you can read the [blog post about the new Request Queue storage system on the Apify blog](https://blog.apify.com/new-apify-request-queue/). ::: From 79d442cf191f3bc615bb2abacefacbafd22784b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Mon, 13 May 2024 16:46:40 +0200 Subject: [PATCH 12/12] Update docs/experiments/request_locking.mdx --- docs/experiments/request_locking.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/experiments/request_locking.mdx b/docs/experiments/request_locking.mdx index b1608e5d9ca..224b59ec700 100644 --- a/docs/experiments/request_locking.mdx +++ b/docs/experiments/request_locking.mdx @@ -8,7 +8,7 @@ import ApiLink from '@site/src/components/ApiLink'; :::tip Release announcement -As of **May 2024** (`crawlee` version `3.9.0`), this experiment is now enabled by default! With that said, if you encounter issues you can: +As of **May 2024** (`crawlee` version `3.10.0`), this experiment is now enabled by default! With that said, if you encounter issues you can: - set `requestLocking` to `false` in the `experiments` object of your crawler options - update all imports of `RequestQueue` to `RequestQueueV1`