Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 25 additions & 26 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,47 +54,46 @@
]
},
"devDependencies": {
"@apify/consts": "^2.29.0",
"@apify/input_secrets": "^1.2.0",
"@apify/log": "^2.5.22",
"@apify/consts": "^2.57.0",
"@apify/input_secrets": "^1.2.54",
"@apify/log": "^2.5.48",
"@apify/oxlint-config": "^0.3.0",
"@apify/tsconfig": "^0.2.0",
"@commitlint/config-conventional": "^21.0.0",
"@commitlint/config-conventional": "^21.2.0",
"@isaacs/brace-expansion": "^5.0.1",
"@playwright/browser-chromium": "^1.61.0",
"@types/content-type": "^1.1.8",
"@playwright/browser-chromium": "^1.62.1",
"@types/fs-extra": "^11.0.4",
"@types/node": "^24.0.0",
"@types/semver": "^7.5.8",
"@types/node": "^24.13.3",
"@types/semver": "^7.8.0",
"@types/tough-cookie": "^4.0.5",
"@types/ws": "^8.5.12",
"apify": "^3.7.1",
"apify-client": "^2.17.0",
"commitlint": "^21.0.0",
"crawlee": "^3.16.0",
"fs-extra": "^11.2.0",
"@types/ws": "^8.18.1",
"apify": "^3.7.2",
"apify-client": "^2.24.0",
"commitlint": "^21.2.1",
"crawlee": "^3.18.0",
"fs-extra": "^11.4.0",
"gen-esm-wrapper": "^1.1.3",
"globby": "^14.1.0",
"husky": "^9.1.7",
"lerna": "^9.0.0",
"lint-staged": "^16.0.0",
"lerna": "^9.0.7",
"lint-staged": "^17.3.0",
"oxfmt": "0.61.0",
"oxlint": "1.73.0",
"oxlint-tsgolint": "0.24.0",
"playwright": "^1.61.0",
"oxlint": "1.77.0",
"oxlint-tsgolint": "7.0.2001",
"playwright": "^1.62.1",
"puppeteer": "25.4.0",
"rimraf": "^6.0.1",
"tsx": "^4.16.5",
"turbo": "2.10.4",
"typescript": "~6.0.0",
"vite-tsconfig-paths": "^6.0.0",
"rimraf": "^6.1.3",
"tsx": "^4.23.5",
"turbo": "2.10.8",
"typescript": "~6.0.3",
"vite-tsconfig-paths": "^6.1.1",
"vitest": "^4.1.10"
},
"packageManager": "pnpm@10.33.4",
"packageManager": "pnpm@11.20.0",
"devEngines": {
"packageManager": {
"name": "pnpm",
"version": "10.33.4",
"version": "11.20.0",
"onFail": "warn"
}
}
Expand Down
18 changes: 11 additions & 7 deletions packages/actor-scraper/camoufox-scraper/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
# Build context is the monorepo root (dockerContextDir in .actor/actor.json), so the build uses the
# root pnpm-lock.yaml for a deterministic install.
#
# The base is pinned by digest (reproducible Camoufox binary + OS deps). The actor pins playwright and
# camoufox-js to the versions this base ships (24-1.59.1 -> playwright 1.59.1, camoufox-js 0.11.1) and
# bundles them, so the bundled drivers match the Camoufox binary they drive. The binary itself lives in
# ~/.cache/camoufox (outside node_modules) and stays in the base. Tag 24-1.59.1 is the newest Camoufox
# base that survives page JS errors (24-1.60.0 crashes the process on them).
FROM apify/actor-node-playwright-camoufox:24-1.59.1@sha256:eadc96fa9492284eb45ef70b6b91c841fae7f142d25a22ae2887a21bb78b3469 AS builder
# The base is pinned by digest (reproducible Camoufox binary + OS deps). The actor pins playwright to
# the version this base ships (24-1.59.1 -> playwright 1.59.1) and bundles it, so the bundled driver
# matches the Camoufox binary it drives, which is why this actor deliberately trails the playwright
# version the rest of the workspace uses. The binary itself lives in ~/.cache/camoufox (outside
# node_modules) and stays in the base. Tag 1.59.1 is the newest Camoufox base that survives page JS
# errors (1.60.0 crashes the process on them).
#
# camoufox-js stays on 0.11.2: 0.12.0 requires better-sqlite3 ^13, which dropped prebuilt binaries and
# so needs a C++ toolchain that the Apify base images do not carry.
FROM apify/actor-node-playwright-camoufox:24-1.59.1@sha256:02cf108662749342538908f9409ffc270b741af1253e96d0c563b12de36f3b88 AS builder

# The browser base runs as `myuser`; the build stage needs root for corepack + a writable workdir.
USER root
Expand All @@ -32,7 +36,7 @@ RUN pnpm --filter actor-camoufox-scraper... build
# compiled here against the base's Node 24, and the runtime stage uses that same base, so it matches.
RUN pnpm --config.inject-workspace-packages=true --filter actor-camoufox-scraper deploy --prod /deploy

FROM apify/actor-node-playwright-camoufox:24-1.59.1@sha256:eadc96fa9492284eb45ef70b6b91c841fae7f142d25a22ae2887a21bb78b3469
FROM apify/actor-node-playwright-camoufox:24-1.59.1@sha256:02cf108662749342538908f9409ffc270b741af1253e96d0c563b12de36f3b88

# Base WORKDIR is /home/myuser and ships the Xvfb entrypoint plus the Camoufox binary in
# ~/.cache/camoufox (outside node_modules). Replace only the template node_modules with the lean
Expand Down
20 changes: 10 additions & 10 deletions packages/actor-scraper/camoufox-scraper/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,21 @@
"description": "Crawl web pages using Apify, headless browser and Playwright with Camoufox",
"type": "module",
"dependencies": {
"@apify/scraper-tools": "^1.1.4",
"@crawlee/core": "^3.14.1",
"@crawlee/playwright": "^3.14.1",
"@crawlee/utils": "^3.14.1",
"apify": "^3.7.1",
"@apify/scraper-tools": "workspace:^",
"@crawlee/core": "^3.18.0",
"@crawlee/playwright": "^3.18.0",
"@crawlee/utils": "^3.18.0",
"apify": "^3.7.2",
"camoufox-js": "0.11.2",
"idcac-playwright": "^0.2.0",
"playwright": "1.61.1",
"playwright-core": "1.61.1"
"playwright": "1.59.1",
"playwright-core": "1.59.1"
},
"devDependencies": {
"@apify/tsconfig": "^0.2.0",
"@types/node": "^24.0.0",
"tsx": "^4.19.1",
"typescript": "~6.0.0"
"@types/node": "^24.13.3",
"tsx": "^4.23.5",
"typescript": "~6.0.3"
},
"scripts": {
"start": "pnpm start:dev",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ export class CrawlerSetup implements CrawlerSetupOptions {
}),
// `firefox` and @crawlee/playwright's PlaywrightLaunchContext resolve to two different
// playwright-core copies (camoufox pins playwright 1.59.1 to match its base image,
// while the rest of the workspace is on 1.61.0), so the BrowserType identities don't
// while the rest of the workspace is on 1.62.1), so the BrowserType identities don't
// line up. This is a workspace dual-install, not a camoufox-js/crawlee issue.
} as unknown as PlaywrightLaunchContext,
useSessionPool: true,
Expand Down
12 changes: 6 additions & 6 deletions packages/actor-scraper/cheerio-scraper/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
"description": "Crawl web pages using HTTP requests and Cheerio",
"type": "module",
"dependencies": {
"@apify/scraper-tools": "^1.1.4",
"@crawlee/cheerio": "^3.16.0",
"apify": "^3.7.1"
"@apify/scraper-tools": "workspace:^",
"@crawlee/cheerio": "^3.18.0",
"apify": "^3.7.2"
},
"devDependencies": {
"@apify/tsconfig": "^0.2.0",
"@types/node": "^24.0.0",
"tsx": "^4.19.1",
"typescript": "~6.0.0"
"@types/node": "^24.13.3",
"tsx": "^4.23.5",
"typescript": "~6.0.3"
},
"peerDependencies": {
"cheerio": "^1.0.0-rc.12"
Expand Down
14 changes: 7 additions & 7 deletions packages/actor-scraper/jsdom-scraper/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,18 @@
"description": "Crawl web pages using HTTP requests and JSDOM parser",
"type": "module",
"dependencies": {
"@apify/scraper-tools": "^1.1.4",
"@crawlee/jsdom": "^3.14.1",
"apify": "^3.7.1"
"@apify/scraper-tools": "workspace:^",
"@crawlee/jsdom": "^3.18.0",
"apify": "^3.7.2"
},
"devDependencies": {
"@apify/tsconfig": "^0.2.0",
"@types/node": "^24.0.0",
"tsx": "^4.19.1",
"typescript": "~6.0.0"
"@types/node": "^24.13.3",
"tsx": "^4.23.5",
"typescript": "~6.0.3"
},
"peerDependencies": {
"jsdom": "^29.0.0"
"jsdom": "^29.0.0 || ^30.0.0"
},
"scripts": {
"start": "pnpm start:dev",
Expand Down
6 changes: 3 additions & 3 deletions packages/actor-scraper/playwright-scraper/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# root pnpm-lock.yaml for a deterministic install.
#
# The base is pinned by digest (reproducible Chromium + OS deps). The actor pins playwright to the
# version this base ships (24-1.61.0 -> playwright 1.61.0) and bundles it, so the bundled driver
# version this base ships (24-1.62.1 -> playwright 1.62.1) and bundles it, so the bundled driver
# matches the Chromium binary it drives. Renovate bumps the base digest and the playwright pin together.
FROM apify/actor-node-playwright:24-1.61.0@sha256:5c0155bb5a0a90572ef021c2b4e2e195c50971e0c1ae00f081ffa47e62036c60 AS builder
FROM apify/actor-node-playwright:24-1.62.1@sha256:69172fc9daa6c6e7d3d1e57491c91852a0d6af2818378d341f65e69795e68ab4 AS builder

# The browser base runs as `myuser`; the build stage needs root for corepack + a writable workdir.
USER root
Expand All @@ -29,7 +29,7 @@ RUN pnpm --filter actor-playwright-scraper... build
# into node_modules instead of symlinking it.
RUN pnpm --config.inject-workspace-packages=true --filter actor-playwright-scraper deploy --prod /deploy

FROM apify/actor-node-playwright:24-1.61.0@sha256:5c0155bb5a0a90572ef021c2b4e2e195c50971e0c1ae00f081ffa47e62036c60
FROM apify/actor-node-playwright:24-1.62.1@sha256:69172fc9daa6c6e7d3d1e57491c91852a0d6af2818378d341f65e69795e68ab4

# Base WORKDIR is /home/myuser and ships the browser plus the Xvfb entrypoint. Replace only the
# template node_modules with the lean production bundle (which carries the matching playwright); the
Expand Down
18 changes: 9 additions & 9 deletions packages/actor-scraper/playwright-scraper/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,19 @@
"description": "Crawl web pages using Apify, headless browser and Playwright",
"type": "module",
"dependencies": {
"@apify/scraper-tools": "^1.1.4",
"@crawlee/core": "^3.16.0",
"@crawlee/playwright": "^3.16.0",
"@crawlee/utils": "^3.16.0",
"apify": "^3.7.1",
"@apify/scraper-tools": "workspace:^",
"@crawlee/core": "^3.18.0",
"@crawlee/playwright": "^3.18.0",
"@crawlee/utils": "^3.18.0",
"apify": "^3.7.2",
"idcac-playwright": "^0.2.0",
"playwright": "1.61.1"
"playwright": "1.62.1"
},
"devDependencies": {
"@apify/tsconfig": "^0.2.0",
"@types/node": "^24.0.0",
"tsx": "^4.19.1",
"typescript": "~6.0.0"
"@types/node": "^24.13.3",
"tsx": "^4.23.5",
"typescript": "~6.0.3"
},
"scripts": {
"start": "pnpm start:dev",
Expand Down
11 changes: 6 additions & 5 deletions packages/actor-scraper/puppeteer-scraper/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# root pnpm-lock.yaml for a deterministic install.
#
# The base is pinned by digest (reproducible Chrome + OS deps). The actor pins puppeteer to the version
# this base ships (24-25.2.0 -> puppeteer 25.2.0) and bundles it, so the bundled driver matches the
# this base ships (24-25.4.0 -> puppeteer 25.4.0) and bundles it, so the bundled driver matches the
# Chrome binary it drives. Renovate bumps the base digest and the puppeteer pin together.
FROM apify/actor-node-puppeteer-chrome:24-25.2.0@sha256:e9f588cd827e542bdc7b476be7dd3ba87b8ca3b4136e2e081e5fa0e3c940d50d AS builder
FROM apify/actor-node-puppeteer-chrome:24-25.4.0@sha256:b8efd3ada1cb332815db68a6228c00c8fb19f66e8a8305225360c99c4b853e62 AS builder

# The browser base runs as `myuser`; the build stage needs root for corepack + a writable workdir.
USER root
Expand All @@ -29,16 +29,17 @@ RUN pnpm --filter actor-puppeteer-scraper... build
# into node_modules instead of symlinking it.
RUN pnpm --config.inject-workspace-packages=true --filter actor-puppeteer-scraper deploy --prod /deploy

FROM apify/actor-node-puppeteer-chrome:24-25.2.0@sha256:e9f588cd827e542bdc7b476be7dd3ba87b8ca3b4136e2e081e5fa0e3c940d50d
FROM apify/actor-node-puppeteer-chrome:24-25.4.0@sha256:b8efd3ada1cb332815db68a6228c00c8fb19f66e8a8305225360c99c4b853e62

# Base WORKDIR is /home/myuser and ships Chrome plus the Xvfb entrypoint. Replace only the template
# node_modules with the lean production bundle (which carries the matching puppeteer); Chrome itself
# stays in the base image.
RUN rm -rf node_modules
COPY --from=builder --chown=myuser /deploy ./

# The base announces its Chrome via APIFY_CHROME_EXECUTABLE_PATH, but apify@3.7/crawlee@3.16 read only
# PUPPETEER_EXECUTABLE_PATH — mirror the base's own value so we follow whatever Chrome the base ships.
# The base announces its Chrome via APIFY_CHROME_EXECUTABLE_PATH, but the apify 3.x / crawlee 3.x
# launcher reads only PUPPETEER_EXECUTABLE_PATH — mirror the base's own value so we follow whatever
# Chrome the base ships.
ENV PUPPETEER_EXECUTABLE_PATH=${APIFY_CHROME_EXECUTABLE_PATH} \
APIFY_DISABLE_OUTDATED_WARNING=1

Expand Down
12 changes: 6 additions & 6 deletions packages/actor-scraper/puppeteer-scraper/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@
"description": "Crawl web pages using Apify, headless Chrome and Puppeteer",
"type": "module",
"dependencies": {
"@apify/scraper-tools": "^1.1.4",
"@crawlee/puppeteer": "^3.16.0",
"apify": "^3.7.1",
"@apify/scraper-tools": "workspace:^",
"@crawlee/puppeteer": "^3.18.0",
"apify": "^3.7.2",
"idcac-playwright": "^0.2.0",
"puppeteer": "25.4.0"
},
"devDependencies": {
"@apify/tsconfig": "^0.2.0",
"@types/node": "^24.0.0",
"tsx": "^4.19.1",
"typescript": "~6.0.0"
"@types/node": "^24.13.3",
"tsx": "^4.23.5",
"typescript": "~6.0.3"
},
"scripts": {
"start": "pnpm start:dev",
Expand Down
8 changes: 4 additions & 4 deletions packages/actor-scraper/sitemap-scraper/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"description": "Crawl web pages from a sitemap using HTTP HEAD requests",
"type": "module",
"dependencies": {
"@apify/scraper-tools": "^1.1.4",
"@apify/scraper-tools": "workspace:^",
"@crawlee/core": "4.0.0-beta.25",
"@crawlee/http": "4.0.0-beta.25",
"@crawlee/impit-client": "4.0.0-beta.29",
Expand All @@ -20,9 +20,9 @@
},
"devDependencies": {
"@apify/tsconfig": "^0.2.0",
"@types/node": "^24.0.0",
"tsx": "^4.19.1",
"typescript": "~6.0.0"
"@types/node": "^24.13.3",
"tsx": "^4.23.5",
"typescript": "~6.0.3"
},
"scripts": {
"start": "pnpm start:dev",
Expand Down
11 changes: 6 additions & 5 deletions packages/actor-scraper/web-scraper/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# root pnpm-lock.yaml for a deterministic install.
#
# The base is pinned by digest (reproducible Chrome + OS deps). The actor pins puppeteer to the version
# this base ships (24-25.2.0 -> puppeteer 25.2.0) and bundles it, so the bundled driver matches the
# this base ships (24-25.4.0 -> puppeteer 25.4.0) and bundles it, so the bundled driver matches the
# Chrome binary it drives. Renovate bumps the base digest and the puppeteer pin together.
FROM apify/actor-node-puppeteer-chrome:24-25.2.0@sha256:e9f588cd827e542bdc7b476be7dd3ba87b8ca3b4136e2e081e5fa0e3c940d50d AS builder
FROM apify/actor-node-puppeteer-chrome:24-25.4.0@sha256:b8efd3ada1cb332815db68a6228c00c8fb19f66e8a8305225360c99c4b853e62 AS builder

# The browser base runs as `myuser`; the build stage needs root for corepack + a writable workdir.
USER root
Expand All @@ -29,16 +29,17 @@ RUN pnpm --filter actor-web-scraper... build
# into node_modules instead of symlinking it.
RUN pnpm --config.inject-workspace-packages=true --filter actor-web-scraper deploy --prod /deploy

FROM apify/actor-node-puppeteer-chrome:24-25.2.0@sha256:e9f588cd827e542bdc7b476be7dd3ba87b8ca3b4136e2e081e5fa0e3c940d50d
FROM apify/actor-node-puppeteer-chrome:24-25.4.0@sha256:b8efd3ada1cb332815db68a6228c00c8fb19f66e8a8305225360c99c4b853e62

# Base WORKDIR is /home/myuser and ships Chrome plus the Xvfb entrypoint. Replace only the template
# node_modules with the lean production bundle (which carries the matching puppeteer); Chrome itself
# stays in the base image.
RUN rm -rf node_modules
COPY --from=builder --chown=myuser /deploy ./

# The base announces its Chrome via APIFY_CHROME_EXECUTABLE_PATH, but apify@3.7/crawlee@3.16 read only
# PUPPETEER_EXECUTABLE_PATH — mirror the base's own value so we follow whatever Chrome the base ships.
# The base announces its Chrome via APIFY_CHROME_EXECUTABLE_PATH, but the apify 3.x / crawlee 3.x
# launcher reads only PUPPETEER_EXECUTABLE_PATH — mirror the base's own value so we follow whatever
# Chrome the base ships.
ENV PUPPETEER_EXECUTABLE_PATH=${APIFY_CHROME_EXECUTABLE_PATH} \
APIFY_DISABLE_OUTDATED_WARNING=1

Expand Down
15 changes: 7 additions & 8 deletions packages/actor-scraper/web-scraper/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,19 @@
"main": "dist/main.js",
"type": "module",
"dependencies": {
"@apify/scraper-tools": "^1.1.4",
"@crawlee/puppeteer": "^3.16.0",
"apify": "^3.7.1",
"content-type": "^1.0.5",
"@apify/scraper-tools": "workspace:^",
"@crawlee/puppeteer": "^3.18.0",
"apify": "^3.7.2",
"content-type": "^2.0.0",
"devtools-server": "^0.0.2",
"idcac-playwright": "^0.2.0",
"puppeteer": "25.4.0"
},
"devDependencies": {
"@apify/tsconfig": "^0.2.0",
"@types/content-type": "^1.1.8",
"@types/node": "^24.0.0",
"tsx": "^4.19.1",
"typescript": "~6.0.0"
"@types/node": "^24.13.3",
"tsx": "^4.23.5",
"typescript": "~6.0.3"
},
"scripts": {
"start": "pnpm start:dev",
Expand Down
Loading
Loading