Skip to content

Commit

Permalink
fix: pin all internal dependencies (#2041)
Browse files Browse the repository at this point in the history 
Related: #2040
  • Loading branch information
@B4nan
B4nan committed Aug 21, 2023
1 parent 3741578 commit d6f2b17
Show file tree
Hide file tree
Showing 18 changed files with 129 additions and 109 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
View file
Expand Up @@ -137,8 +137,9 @@ jobs:
GIT_USER: "noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

- name: Update lockfile
- name: Pin versions in internal dependencies and update lockfile
run: |
yarn release:tilde
yarn install --no-immutable
git add .
git diff-index --quiet HEAD || git commit -m 'chore(release): update internal dependencies [skip ci]'
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Expand Up @@ -46,6 +46,7 @@
"release:next": "yarn build && yarn publish:next",
"publish:prod": "lerna publish from-package --contents dist --force-publish",
"release:prod": "yarn build && yarn publish:prod",
"release:pin-versions": "turbo run copy -- -- --pin-versions",
"lint": "eslint \"packages/**/*.ts\" \"test/**/*.ts\"",
"lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.ts\" --fix"
},
Expand Down
6 changes: 3 additions & 3 deletions packages/basic-crawler/package.json
View file
Expand Up @@ -48,9 +48,9 @@
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
"@crawlee/core": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/utils": "^3.5.2",
"@crawlee/core": "3.5.2",
"@crawlee/types": "3.5.2",
"@crawlee/utils": "3.5.2",
"got-scraping": "^3.2.9",
"ow": "^0.28.1",
"tldts": "^6.0.0",
Expand Down
8 changes: 4 additions & 4 deletions packages/browser-crawler/package.json
View file
Expand Up @@ -54,10 +54,10 @@
},
"dependencies": {
"@apify/timeout": "^0.3.0",
"@crawlee/basic": "^3.5.2",
"@crawlee/browser-pool": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/utils": "^3.5.2",
"@crawlee/basic": "3.5.2",
"@crawlee/browser-pool": "3.5.2",
"@crawlee/types": "3.5.2",
"@crawlee/utils": "3.5.2",
"ow": "^0.28.1",
"tslib": "^2.4.0"
}
Expand Down
2 changes: 1 addition & 1 deletion packages/browser-pool/package.json
View file
Expand Up @@ -38,7 +38,7 @@
"dependencies": {
"@apify/log": "^2.4.0",
"@apify/timeout": "^0.3.0",
"@crawlee/types": "^3.5.2",
"@crawlee/types": "3.5.2",
"fingerprint-generator": "^2.0.6",
"fingerprint-injector": "^2.0.5",
"lodash.merge": "^4.6.2",
Expand Down
4 changes: 2 additions & 2 deletions packages/cheerio-crawler/package.json
View file
Expand Up @@ -53,8 +53,8 @@
"access": "public"
},
"dependencies": {
"@crawlee/http": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/http": "3.5.2",
"@crawlee/types": "3.5.2",
"cheerio": "^1.0.0-rc.12",
"htmlparser2": "^9.0.0",
"tslib": "^2.4.0"
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/package.json
View file
Expand Up @@ -51,7 +51,7 @@
"access": "public"
},
"dependencies": {
"@crawlee/templates": "^3.5.2",
"@crawlee/templates": "3.5.2",
"ansi-colors": "^4.1.3",
"fs-extra": "^11.0.0",
"inquirer": "^8.2.4",
Expand Down
6 changes: 3 additions & 3 deletions packages/core/package.json
View file
Expand Up @@ -59,9 +59,9 @@
"@apify/pseudo_url": "^2.0.30",
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
"@crawlee/memory-storage": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/utils": "^3.5.2",
"@crawlee/memory-storage": "3.5.2",
"@crawlee/types": "3.5.2",
"@crawlee/utils": "3.5.2",
"@sapphire/async-queue": "^1.5.0",
"@types/tough-cookie": "^4.0.2",
"@vladfrangu/async_event_emitter": "^2.2.2",
Expand Down
24 changes: 12 additions & 12 deletions packages/crawlee/package.json
View file
Expand Up @@ -54,18 +54,18 @@
"access": "public"
},
"dependencies": {
"@crawlee/basic": "^3.5.2",
"@crawlee/browser": "^3.5.2",
"@crawlee/browser-pool": "^3.5.2",
"@crawlee/cheerio": "^3.5.2",
"@crawlee/cli": "^3.5.2",
"@crawlee/core": "^3.5.2",
"@crawlee/http": "^3.5.2",
"@crawlee/jsdom": "^3.5.2",
"@crawlee/linkedom": "^3.5.2",
"@crawlee/playwright": "^3.5.2",
"@crawlee/puppeteer": "^3.5.2",
"@crawlee/utils": "^3.5.2",
"@crawlee/basic": "3.5.2",
"@crawlee/browser": "3.5.2",
"@crawlee/browser-pool": "3.5.2",
"@crawlee/cheerio": "3.5.2",
"@crawlee/cli": "3.5.2",
"@crawlee/core": "3.5.2",
"@crawlee/http": "3.5.2",
"@crawlee/jsdom": "3.5.2",
"@crawlee/linkedom": "3.5.2",
"@crawlee/playwright": "3.5.2",
"@crawlee/puppeteer": "3.5.2",
"@crawlee/utils": "3.5.2",
"import-local": "^3.1.0",
"tslib": "^2.4.0"
},
Expand Down
6 changes: 3 additions & 3 deletions packages/http-crawler/package.json
View file
Expand Up @@ -55,9 +55,9 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
"@crawlee/basic": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/utils": "^3.5.2",
"@crawlee/basic": "3.5.2",
"@crawlee/types": "3.5.2",
"@crawlee/utils": "3.5.2",
"@types/content-type": "^1.1.5",
"cheerio": "^1.0.0-rc.12",
"content-type": "^1.0.4",
Expand Down
4 changes: 2 additions & 2 deletions packages/jsdom-crawler/package.json
View file
Expand Up @@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
"@crawlee/http": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/http": "3.5.2",
"@crawlee/types": "3.5.2",
"@types/jsdom": "^21.0.0",
"cheerio": "^1.0.0-rc.12",
"jsdom": "^22.0.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/linkedom-crawler/package.json
View file
Expand Up @@ -55,8 +55,8 @@
"dependencies": {
"@apify/timeout": "^0.3.0",
"@apify/utilities": "^2.7.10",
"@crawlee/http": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/http": "3.5.2",
"@crawlee/types": "3.5.2",
"linkedom": "^0.15.0",
"ow": "^0.28.2",
"tslib": "^2.4.0"
Expand Down
2 changes: 1 addition & 1 deletion packages/memory-storage/package.json
View file
Expand Up @@ -49,7 +49,7 @@
},
"dependencies": {
"@apify/log": "^2.4.0",
"@crawlee/types": "^3.5.2",
"@crawlee/types": "3.5.2",
"@sapphire/async-queue": "^1.5.0",
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
Expand Down
8 changes: 4 additions & 4 deletions packages/playwright-crawler/package.json
View file
Expand Up @@ -55,10 +55,10 @@
"dependencies": {
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.4.0",
"@crawlee/browser": "^3.5.2",
"@crawlee/browser-pool": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/utils": "^3.5.2",
"@crawlee/browser": "3.5.2",
"@crawlee/browser-pool": "3.5.2",
"@crawlee/types": "3.5.2",
"@crawlee/utils": "3.5.2",
"cheerio": "^1.0.0-rc.12",
"idcac-playwright": "^0.1.2",
"jquery": "^3.6.0",
Expand Down
8 changes: 4 additions & 4 deletions packages/puppeteer-crawler/package.json
View file
Expand Up @@ -55,10 +55,10 @@
"dependencies": {
"@apify/datastructures": "^2.0.0",
"@apify/log": "^2.4.0",
"@crawlee/browser": "^3.5.2",
"@crawlee/browser-pool": "^3.5.2",
"@crawlee/types": "^3.5.2",
"@crawlee/utils": "^3.5.2",
"@crawlee/browser": "3.5.2",
"@crawlee/browser-pool": "3.5.2",
"@crawlee/types": "3.5.2",
"@crawlee/utils": "3.5.2",
"cheerio": "^1.0.0-rc.12",
"devtools-protocol": "*",
"idcac-playwright": "^0.1.2",
Expand Down
2 changes: 1 addition & 1 deletion packages/utils/package.json
View file
Expand Up @@ -49,7 +49,7 @@
"dependencies": {
"@apify/log": "^2.4.0",
"@apify/ps-tree": "^1.2.0",
"@crawlee/types": "^3.5.2",
"@crawlee/types": "3.5.2",
"cheerio": "^1.0.0-rc.12",
"got-scraping": "^3.2.9",
"ow": "^0.28.1",
Expand Down
32 changes: 25 additions & 7 deletions scripts/copy.ts
View file
@@ -1,6 +1,7 @@
import { copyFileSync, readFileSync, writeFileSync } from 'fs';
import { resolve } from 'path';
/* eslint-disable @typescript-eslint/no-var-requires,import/no-dynamic-require,global-require */
import { execSync } from 'node:child_process';
import { copyFileSync, readFileSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';

const options = process.argv.slice(2).reduce((args, arg) => {
const [key, value] = arg.split('=');
Expand All @@ -25,17 +26,19 @@ function rewrite(path: string, replacer: (from: string) => string): void {

let rootVersion: string;

function getRootVersion(): string {
function getRootVersion(bump = true): string {
if (rootVersion) {
return rootVersion;
}

// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-dynamic-require,global-require
rootVersion = require(resolve(root, './lerna.json')).version.replace(/^(\d+\.\d+\.\d+)-?.*$/, '$1');

const parts = rootVersion.split('.');
parts[2] = `${+parts[2] + 1}`;
rootVersion = parts.join('.');
if (bump) {
const parts = rootVersion.split('.');
parts[2] = `${+parts[2] + 1}`;
rootVersion = parts.join('.');
}

return rootVersion;
}
Expand Down Expand Up @@ -79,7 +82,6 @@ const target = resolve(process.cwd(), 'dist');
const pkgPath = resolve(process.cwd(), 'package.json');

if (options.canary) {
// eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-dynamic-require,global-require
const pkgJson = require(pkgPath);
const nextVersion = getNextVersion();
pkgJson.version = nextVersion;
Expand All @@ -97,6 +99,22 @@ if (options.canary) {
writeFileSync(pkgPath, `${JSON.stringify(pkgJson, null, 4)}\n`);
}

if (options['pin-versions']) {
const pkgJson = require(pkgPath);
const version = getRootVersion(false);

for (const dep of Object.keys(pkgJson.dependencies ?? {})) {
if (dep.startsWith('@crawlee/') || dep === 'crawlee') {
pkgJson.dependencies[dep] = version;
}
}

// eslint-disable-next-line no-console
console.info(`pin-versions: version ${version}`, pkgJson.dependencies);

writeFileSync(pkgPath, `${JSON.stringify(pkgJson, null, 4)}\n`);
}

copy('README.md', root, target);
copy('LICENSE.md', root, target);
copy('package.json', process.cwd(), target);
Expand Down

0 comments on commit d6f2b17

Please sign in to comment.