refactor: convert to native ESM

BREAKING CHANGE:

The project is now native ESM without a CJS alternative. This is fine since all supported node versions allow `require(esm)`.

Also all the dependencies are updated to the latest versions, including cheerio v1.

Author: B4nan

docs/upgrading/upgrading_v4.md

@@ -0,0 +1,24 @@
+---
+id: upgrading-to-v4
+title: Upgrading to v4
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+
+This page summarizes most of the breaking changes in Crawlee v4.
+
+## ECMAScript modules
+
+Crawlee v4 is a native ESM package now. It can be still consumed from a CJS project, as long as you use TypeScript and Node.js version that supports `require(esm)`.
+
+## Node 22+ required
+
+Support for older node versions was dropped.
+
+## TypeScript 5.8+ required
+
+Support for older TypeScript versions was dropped. Older versions might work too, but only if your project is also ESM.
+
+## Cheerio v1
+
+Previously, we kept the dependency on cheerio locked to the latest RC version, since there were many breaking changes introduced in v1.0. This release bumps cheerio to the stable v1. Also, we now use the default `parse5` internally.

eslint.config.mjs

@@ -1,3 +1,6 @@
+import tsEslint from 'typescript-eslint';
+import tsStylistic from '@stylistic/eslint-plugin-ts';
+import apifyJs from '@apify/eslint-config/js';
 import apify from '@apify/eslint-config/ts';
 import stylistic from '@stylistic/eslint-plugin';
 import prettier from 'eslint-config-prettier';
@@ -78,4 +81,13 @@ export default [
             'no-undef': 'off',
         },
     },
+    // {
+    //     files: ['test/**/*'],
+    //     rules: {
+    //         ...apifyJs.rules,
+    //         '@typescript-eslint/no-floating-promises': 'off',
+    //         'no-console': 'off',
+    //         'no-undef': 'off',
+    //     },
+    // },
 ];

package.json

@@ -1,6 +1,7 @@
 {
     "name": "@crawlee/root",
     "private": true,
+    "type": "module",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "workspaces": [
         "packages/*"
@@ -49,70 +50,73 @@
         "release:prod": "yarn build && yarn publish:prod",
         "release:pin-versions": "turbo run copy -- -- --pin-versions",
         "lint": "eslint \"packages/**/*.ts\" \"test/**/*.ts\"",
-        "lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.ts\" --fix",
+        "lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.{ts,mjs}\" --fix",
         "format": "biome format --write .",
         "format:check": "biome format .",
         "prepare": "husky"
     },
     "devDependencies": {
         "@apify/eslint-config": "2.0.0",
-        "@apify/log": "^2.4.0",
+        "@apify/log": "^2.5.18",
         "@apify/tsconfig": "^0.2.0",
         "@biomejs/biome": "^2.3.11",
         "@commitlint/config-conventional": "^21.0.0",
-        "@playwright/browser-chromium": "1.62.1",
-        "@playwright/browser-firefox": "1.62.1",
-        "@playwright/browser-webkit": "1.62.1",
+        "@playwright/browser-chromium": "1.61.1",
+        "@playwright/browser-firefox": "1.61.1",
+        "@playwright/browser-webkit": "1.61.1",
         "@stylistic/eslint-plugin": "5.7.1",
-        "@types/content-type": "^1.1.5",
-        "@types/deep-equal": "^1.0.1",
-        "@types/domhandler": "^2.4.2",
-        "@types/express": "^5.0.0",
-        "@types/fs-extra": "^11.0.0",
-        "@types/inquirer": "^8.2.1",
-        "@types/is-ci": "^3.0.1",
+        "@stylistic/eslint-plugin-ts": "^4.2.0",
+        "@types/content-type": "^1.1.8",
+        "@types/deep-equal": "^1.0.4",
+        "@types/domhandler": "^3.1.0",
+        "@types/express": "^5.0.1",
+        "@types/fs-extra": "^11.0.4",
+        "@types/inquirer": "^9.0.8",
+        "@types/is-ci": "^3.0.4",
         "@types/lodash.isequal": "^4.5.8",
-        "@types/lodash.merge": "^4.6.7",
-        "@types/mime-types": "^2.1.1",
+        "@types/lodash.merge": "^4.6.9",
+        "@types/mime-types": "^2.1.4",
         "@types/node": "^24.0.0",
-        "@types/proper-lockfile": "^4.1.2",
-        "@types/ps-tree": "^1.1.2",
-        "@types/rimraf": "^4.0.0",
-        "@types/sax": "^1.0.0",
-        "@types/semver": "^7.3.12",
-        "@types/stream-json": "^1.7.2",
-        "@types/yargs": "^17.0.26",
+        "@types/proper-lockfile": "^4.1.4",
+        "@types/ps-tree": "^1.1.6",
+        "@types/rimraf": "^4.0.5",
+        "@types/sax": "^1.2.7",
+        "@types/semver": "^7.7.0",
+        "@types/stream-json": "^1.7.8",
+        "@types/whatwg-mimetype": "^3.0.2",
+        "@types/yargs": "^17.0.33",
         "@vitest/coverage-v8": "^4.0.16",
         "apify": "*",
-        "apify-node-curl-impersonate": "^1.0.15",
+        "apify-node-curl-impersonate": "^1.0.23",
         "basic-auth-parser": "^0.0.2",
-        "body-parser": "^2.0.0",
-        "camoufox-js": "^0.12.0",
+        "body-parser": "^2.2.0",
+        "camoufox-js": "^0.11.0",
         "commitlint": "^21.0.0",
         "cross-env": "^10.0.0",
-        "deep-equal": "^2.0.5",
+        "deep-equal": "^2.2.3",
         "eslint": "9.39.1",
         "eslint-config-prettier": "10.1.8",
-        "express": "^4.18.1",
-        "fs-extra": "^11.0.0",
+        "express": "^5.1.0",
+        "fs-extra": "^11.3.0",
         "gen-esm-wrapper": "^1.1.3",
         "globals": "^17.0.0",
         "globby": "^15.0.0",
-        "got": "^13.0.0",
-        "husky": "^9.0.11",
-        "is-ci": "^4.0.0",
-        "lerna": "^10.0.0",
+        "got": "^14.4.7",
+        "husky": "^9.1.7",
+        "is-ci": "^4.1.0",
+        "lerna": "^9.0.7",
         "lint-staged": "^17.0.0",
-        "nock": "^13.4.0",
-        "playwright": "1.62.1",
+        "nock": "^13.5.6",
+        "playwright": "1.61.1",
         "portastic": "^1.0.1",
-        "proxy": "^1.0.2",
-        "puppeteer": "25.8.0",
-        "rimraf": "^6.0.0",
-        "tsx": "^4.4.0",
-        "turbo": "^2.1.0",
+        "proxy": "^2.2.0",
+        "puppeteer": "25.3.0",
+        "rimraf": "^6.0.1",
+        "tsx": "^4.19.4",
+        "turbo": "^2.5.3",
         "typescript": "^6.0.0",
         "typescript-eslint": "8.46.4",
+        "vite-tsconfig-paths": "^5.1.4",
         "vitest": "^4.0.16",
         "zod": "^4.0.0"
     },
@@ -122,15 +126,14 @@
     "resolutions": {
         "tmp": "^0.2.7",
         "@puppeteer/browsers": "^3.0.4",
-        "playwright-core": "1.62.1",
+        "playwright-core": "1.61.1",
         "form-data": "^4.0.6",
         "tar": "^7.5.16",
-        "lerna/js-yaml": "^4.2.0",
-        "node-gyp": "^12.1.0"
+        "lerna/js-yaml": "^4.2.0"
     },
     "packageManager": "yarn@4.10.3",
     "volta": {
-        "node": "24.19.0",
+        "node": "24.18.0",
         "yarn": "4.10.3"
     }
 }

packages/basic-crawler/package.json

@@ -1,19 +1,13 @@
 {
     "name": "@crawlee/basic",
-    "version": "3.18.1",
+    "version": "3.17.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=16.0.0"
+        "node": ">=22.0.0"
     },
-    "main": "./dist/index.js",
-    "module": "./dist/index.mjs",
-    "types": "./dist/index.d.ts",
+    "type": "module",
     "exports": {
-        ".": {
-            "import": "./dist/index.mjs",
-            "require": "./dist/index.js",
-            "types": "./dist/index.d.ts"
-        },
+        ".": "./dist/index.js",
         "./package.json": "./package.json"
     },
     "author": {
@@ -38,26 +32,26 @@
     "scripts": {
         "build": "yarn clean && yarn compile && yarn copy",
         "clean": "rimraf ./dist",
-        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",
+        "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
     },
     "publishConfig": {
         "access": "public"
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.0",
-        "@apify/log": "^2.4.0",
-        "@apify/timeout": "^0.4.0",
-        "@apify/utilities": "^2.7.10",
-        "@crawlee/core": "3.18.1",
-        "@crawlee/types": "3.18.1",
-        "@crawlee/utils": "3.18.1",
-        "csv-stringify": "^6.2.0",
-        "fs-extra": "^11.0.0",
+        "@apify/log": "^2.5.18",
+        "@apify/timeout": "^0.3.2",
+        "@apify/utilities": "^2.15.5",
+        "@crawlee/core": "3.17.0",
+        "@crawlee/types": "3.17.0",
+        "@crawlee/utils": "3.17.0",
+        "csv-stringify": "^6.5.2",
+        "fs-extra": "^11.3.0",
         "got-scraping": "^4.2.1",
-        "ow": "^0.28.1",
-        "tldts": "^7.0.0",
-        "tslib": "^2.4.0",
-        "type-fest": "^4.0.0"
+        "ow": "^2.0.0",
+        "tldts": "^7.0.6",
+        "tslib": "^2.8.1",
+        "type-fest": "^4.41.0"
     }
 }

packages/basic-crawler/src/index.ts

@@ -1,4 +1,4 @@
 export * from '@crawlee/core';
-export * from './internals/basic-crawler';
-export * from './internals/constants';
+export * from './internals/basic-crawler.js';
+export * from './internals/constants.js';
 export { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils';

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -1,3 +1,4 @@
+import { writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
 
 import type {
@@ -58,7 +59,7 @@ import {
 import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';
 import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS, sleep } from '@crawlee/utils';
 import { stringify } from 'csv-stringify/sync';
-import { ensureDir, writeFile, writeJSON } from 'fs-extra';
+import { ensureDir, writeJSON } from 'fs-extra/esm';
 import ow, { ArgumentError } from 'ow';
 import { getDomain } from 'tldts';
 import type { SetRequired } from 'type-fest';
@@ -69,7 +70,7 @@ import defaultLog, { LogLevel } from '@apify/log';
 import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 
-import { createSendRequest } from './send-request';
+import { createSendRequest } from './send-request.js';
 
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>
     extends CrawlingContext<BasicCrawler, UserData> {

packages/basic-crawler/src/internals/send-request.ts

@@ -5,8 +5,7 @@ import {
     type Request,
     type Session,
 } from '@crawlee/core';
-// @ts-expect-error This throws a compilation error due to got-scraping being ESM only but we only import types, so its alllll gooooood
-import type { GotResponse, Method } from 'got-scraping';
+import type { Method, Response as GotResponse } from 'got-scraping';
 
 /**
  * Prepares a function to be used as the `sendRequest` context helper.
@@ -49,6 +48,6 @@ export function createSendRequest(
         // Fill in body as the last step - `processHttpRequestOptions` may use either `body`, `json` or `form` so we cannot override it beforehand
         requestOptions.body ??= originRequest.payload;
 
-        return httpClient.sendRequest<any>(requestOptions);
+        return httpClient.sendRequest<any>(requestOptions) as unknown as GotResponse<Response>;
     };
 }

packages/basic-crawler/test/batch-add-requests.test.ts

@@ -1,6 +1,6 @@
 import { BasicCrawler } from '@crawlee/basic';
 
-import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator';
+import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js';
 
 describe('BasicCrawler#addRequests with big batch sizes', () => {
     const localStorageEmulator = new MemoryStorageEmulator();

packages/basic-crawler/test/migration.test.ts

@@ -1,8 +1,8 @@
 import type { Log } from '@apify/log';
 import log from '@apify/log';
 
-import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator';
-import { BasicCrawler, RequestList } from '../src/index';
+import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js';
+import { BasicCrawler, RequestList } from '../src/index.js';
 
 const localStorageEmulator = new MemoryStorageEmulator();
 

packages/browser-crawler/package.json

@@ -1,19 +1,13 @@
 {
     "name": "@crawlee/browser",
-    "version": "3.18.1",
+    "version": "3.17.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=16.0.0"
+        "node": ">=22.0.0"
     },
-    "main": "./dist/index.js",
-    "module": "./dist/index.mjs",
-    "types": "./dist/index.d.ts",
+    "type": "module",
     "exports": {
-        ".": {
-            "import": "./dist/index.mjs",
-            "require": "./dist/index.js",
-            "types": "./dist/index.d.ts"
-        },
+        ".": "./dist/index.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -46,21 +40,21 @@
     "scripts": {
         "build": "yarn clean && yarn compile && yarn copy",
         "clean": "rimraf ./dist",
-        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",
+        "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
     },
     "publishConfig": {
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.4.0",
-        "@crawlee/basic": "3.18.1",
-        "@crawlee/browser-pool": "3.18.1",
-        "@crawlee/types": "3.18.1",
-        "@crawlee/utils": "3.18.1",
-        "ow": "^0.28.1",
-        "tslib": "^2.4.0",
-        "type-fest": "^4.0.0"
+        "@apify/timeout": "^0.3.2",
+        "@crawlee/basic": "3.17.0",
+        "@crawlee/browser-pool": "3.17.0",
+        "@crawlee/types": "3.17.0",
+        "@crawlee/utils": "3.17.0",
+        "ow": "^2.0.0",
+        "tslib": "^2.8.1",
+        "type-fest": "^4.41.0"
     },
     "peerDependencies": {
         "playwright": "*",