From 5e4e53b03ffc44f5aca5c7e64c5895ce9f4400fa Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 9 Mar 2022 09:29:04 -0800 Subject: [PATCH 1/5] Add webgpu to the release script --- scripts/release-util.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/release-util.ts b/scripts/release-util.ts index 943a24664d0..8c35760d24f 100755 --- a/scripts/release-util.ts +++ b/scripts/release-util.ts @@ -95,6 +95,11 @@ export const WASM_PHASE: Phase = { deps: ['tfjs-core', 'tfjs-backend-cpu'] }; +export const WEBGPU_PHASE: Phase = { + packages: ['tfjs-backend-webgpu'], + deps: ['tfjs-core', 'tfjs-backend-cpu'], +}; + export const VIS_PHASE: Phase = { packages: ['tfjs-vis'] }; @@ -133,6 +138,13 @@ export const TFJS_RELEASE_UNIT: ReleaseUnit = { ] }; +// TODO(mattsoulanille): Move WEBGPU_PHASE to TFJS_RELEASE_UNIT when webgpu +// is out of alpha. +export const WEBGPU_RELEASE_UNIT: ReleaseUnit = { + name: 'tfjs', + phases: [WEBGPU_PHASE], +}; + export const VIS_RELEASE_UNIT: ReleaseUnit = { name: 'vis', phases: [VIS_PHASE] @@ -160,8 +172,9 @@ export const WEBSITE_RELEASE_UNIT: ReleaseUnit = { }; export const RELEASE_UNITS: ReleaseUnit[] = [ - TFJS_RELEASE_UNIT, VIS_RELEASE_UNIT, REACT_NATIVE_RELEASE_UNIT, - TFLITE_RELEASE_UNIT, AUTOML_RELEASE_UNIT, WEBSITE_RELEASE_UNIT + TFJS_RELEASE_UNIT, WEBGPU_RELEASE_UNIT, VIS_RELEASE_UNIT, + REACT_NATIVE_RELEASE_UNIT, TFLITE_RELEASE_UNIT, AUTOML_RELEASE_UNIT, + WEBSITE_RELEASE_UNIT, ]; export const TMP_DIR = '/tmp/tfjs-release'; From 30983c67f3d2551c61af5eddc39cad566c3f3f30 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 9 Mar 2022 10:11:32 -0800 Subject: [PATCH 2/5] Check package.json for link and file deps before publishing --- scripts/publish-npm.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/publish-npm.ts b/scripts/publish-npm.ts index ba1ad4353d8..7e982551738 100755 --- a/scripts/publish-npm.ts +++ b/scripts/publish-npm.ts @@ -26,6 +26,7 @@ import * as argparse from 'argparse'; import chalk from 'chalk'; import * as shell from 'shelljs'; import {RELEASE_UNITS, question, $, printReleaseUnit, printPhase, getReleaseBranch, checkoutReleaseBranch} from './release-util'; +import * as fs from 'fs'; const TMP_DIR = '/tmp/tfjs-publish'; const BAZEL_PACKAGES = new Set([ @@ -85,6 +86,17 @@ async function main() { const pkg = packages[i]; shell.cd(pkg); + // Check the package.json for 'link:' and 'file:' dependencies. + const packageJson = JSON.parse(fs.readFileSync('package.json') + .toString('utf8')) as {dependencies: Record}; + for (let [dep, depVersion] of Object.entries(packageJson.dependencies)) { + const start = depVersion.slice(0,5); + if (start === 'link:' || start === 'file:') { + throw new Error(`${pkg} has a '${start}' dependency on ${dep}. ` + + 'Refusing to publish.'); + } + } + console.log(chalk.magenta.bold(`~~~ Preparing package ${pkg}~~~`)); console.log(chalk.magenta('~~~ Installing packages ~~~')); // tfjs-node-gpu needs to get some files from tfjs-node. From 50b6495e5829c259184cef820ca355b4f5fd2f2e Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 9 Mar 2022 14:08:36 -0800 Subject: [PATCH 3/5] Allow alpha monorepo packages to publish a different version --- scripts/release-tfjs.ts | 56 ++++++++++++----------- scripts/release-util.ts | 98 +++++++++++++++++++++++++---------------- scripts/release.ts | 17 +------ scripts/tsconfig.json | 1 + 4 files changed, 94 insertions(+), 78 deletions(-) diff --git a/scripts/release-tfjs.ts b/scripts/release-tfjs.ts index ec0284af38b..21203b27273 100644 --- a/scripts/release-tfjs.ts +++ b/scripts/release-tfjs.ts @@ -27,7 +27,7 @@ import * as argparse from 'argparse'; import chalk from 'chalk'; import * as fs from 'fs'; import * as shell from 'shelljs'; -import {TMP_DIR, $, question, makeReleaseDir, createPR, TFJS_RELEASE_UNIT, updateTFJSDependencyVersions} from './release-util'; +import {TMP_DIR, $, question, makeReleaseDir, createPR, TFJS_RELEASE_UNIT, updateTFJSDependencyVersions, ALPHA_RELEASE_UNIT, getMinorUpdateVersion, getPatchUpdateVersion} from './release-util'; const parser = new argparse.ArgumentParser(); @@ -36,13 +36,6 @@ parser.addArgument('--git-protocol', { help: 'Use the git protocol rather than the http protocol when cloning repos.' }); -// Computes the default updated version (does a minor version update). -function getMinorUpdateVersion(version: string): string { - const versionSplit = version.split('.'); - - return [versionSplit[0], +versionSplit[1] + 1, '0'].join('.'); -} - async function main() { const args = parser.parseArgs(); const urlBase = args.git_protocol ? 'git@github.com:' : 'https://github.com/'; @@ -52,11 +45,29 @@ async function main() { // Guess release version from tfjs-core's latest version, with a minor update. const latestVersion = $(`npm view @tensorflow/tfjs-core dist-tags.latest`); const minorUpdateVersion = getMinorUpdateVersion(latestVersion); - let newVersion = minorUpdateVersion; - newVersion = - await question(`New version (leave empty for ${minorUpdateVersion}): `); - if (newVersion === '') { - newVersion = minorUpdateVersion; + const newVersion = await question('New version for monorepo (leave empty for ' + + `${minorUpdateVersion}): `) || minorUpdateVersion; + + // Populate the versions map with new versions for monorepo packages. + const versions = new Map(); + for (const phase of TFJS_RELEASE_UNIT.phases) { + for (const packageName of phase.packages) { + versions.set(packageName, newVersion); + } + } + + // Add versions for alpha monorepo packages, which do not have the same + // version as the other monorepo packages. + for (const phase of ALPHA_RELEASE_UNIT.phases) { + for (const packageName of phase.packages) { + const latestVersion = + $(`npm view @tensorflow/${packageName} dist-tags.latest`); + const minorUpdateVersion = getPatchUpdateVersion(latestVersion); + const newVersion = + await question(`New version for alpha package ${packageName}` + + ` (leave empty for ${minorUpdateVersion}): `) || minorUpdateVersion; + versions.set(packageName, newVersion); + } } // Get release candidate commit. @@ -77,15 +88,10 @@ async function main() { $(`git checkout -b ${releaseBranch} ${commit}`); $(`git push origin ${releaseBranch}`); - // Update version. - const phases = TFJS_RELEASE_UNIT.phases; - - for (let i = 0; i < phases.length; i++) { - const packages = phases[i].packages; - const deps = phases[i].deps || []; - - for (let i = 0; i < packages.length; i++) { - const packageName = packages[i]; + // Update versions in package.json files. + const phases = [...TFJS_RELEASE_UNIT.phases, ...ALPHA_RELEASE_UNIT.phases]; + for (const phase of phases) { + for (const packageName of phase.packages) { shell.cd(packageName); // Update the version. @@ -94,10 +100,10 @@ async function main() { const parsedPkg = JSON.parse(`${pkg}`); console.log(chalk.magenta.bold(`~~~ Processing ${packageName} ~~~`)); + const newVersion = versions.get(packageName); pkg = `${pkg}`.replace( - `"version": "${parsedPkg.version}"`, `"version": "${newVersion}"`); - - pkg = updateTFJSDependencyVersions(deps, pkg, parsedPkg, newVersion); + `"version": "${parsedPkg.version}"`, `"version": "${newVersion}"`); + pkg = updateTFJSDependencyVersions(pkg, versions); fs.writeFileSync(packageJsonPath, pkg); diff --git a/scripts/release-util.ts b/scripts/release-util.ts index 8c35760d24f..9baa3dfda3e 100755 --- a/scripts/release-util.ts +++ b/scripts/release-util.ts @@ -140,8 +140,13 @@ export const TFJS_RELEASE_UNIT: ReleaseUnit = { // TODO(mattsoulanille): Move WEBGPU_PHASE to TFJS_RELEASE_UNIT when webgpu // is out of alpha. -export const WEBGPU_RELEASE_UNIT: ReleaseUnit = { - name: 'tfjs', +// Alpha packages that use monorepo dependencies at the latest version but are +// not yet released at the same version number as the monorepo packages. +// Use this for packages that will be a part of the monorepo in the future. +// The release script will ask for a new version for each phase, and it will +// replace 'link' dependencies with the new monorepo version. +export const ALPHA_RELEASE_UNIT: ReleaseUnit = { + name: 'alpha monorepo packages', phases: [WEBGPU_PHASE], }; @@ -172,7 +177,7 @@ export const WEBSITE_RELEASE_UNIT: ReleaseUnit = { }; export const RELEASE_UNITS: ReleaseUnit[] = [ - TFJS_RELEASE_UNIT, WEBGPU_RELEASE_UNIT, VIS_RELEASE_UNIT, + TFJS_RELEASE_UNIT, ALPHA_RELEASE_UNIT, VIS_RELEASE_UNIT, REACT_NATIVE_RELEASE_UNIT, TFLITE_RELEASE_UNIT, AUTOML_RELEASE_UNIT, WEBSITE_RELEASE_UNIT, ]; @@ -288,46 +293,43 @@ export async function updateDependency( // Update package.json dependencies of tfjs packages. This method is different // than `updateDependency`, it does not rely on published versions, instead it -// assumes all the packages have the same version and use that to update. +// uses a map from packageName to newVersion to update the versions. export function updateTFJSDependencyVersions( - deps: string[], pkg: string, parsedPkg: any, tfjsVersion: string): string { - console.log(chalk.magenta.bold(`~~~ Update dependency versions ~~~`)); - - if (deps != null) { - for (let j = 0; j < deps.length; j++) { - const dep = deps[j]; + pkg: string, versions: Map): string { - // Get the current dependency package version. - let version = ''; - const depNpmName = `@tensorflow/${dep}`; - if (parsedPkg['dependencies'] != null && - parsedPkg['dependencies'][depNpmName] != null) { - version = parsedPkg['dependencies'][depNpmName]; - } else if ( - parsedPkg['peerDependencies'] != null && - parsedPkg['peerDependencies'][depNpmName] != null) { - version = parsedPkg['peerDependencies'][depNpmName]; - } else if ( - parsedPkg['devDependencies'] != null && - parsedPkg['devDependencies'][depNpmName] != null) { - version = parsedPkg['devDependencies'][depNpmName]; - } - if (version == null) { - throw new Error(`No dependency found for ${dep}.`); - } + console.log(chalk.magenta.bold(`~~~ Update dependency versions ~~~`)); - let relaxedVersionPrefix = ''; - if (version.startsWith('~') || version.startsWith('^')) { - relaxedVersionPrefix = version.substr(0, 1); - } - const versionLatest = relaxedVersionPrefix + tfjsVersion; + const parsedPkg = JSON.parse(`${pkg}`);JSON.parse(pkg); + for (const [dep, newVersion] of versions) { + // Get the current dependency package version. + let version = ''; + const depNpmName = `@tensorflow/${dep}`; + if (parsedPkg['dependencies'] != null && + parsedPkg['dependencies'][depNpmName] != null) { + version = parsedPkg['dependencies'][depNpmName]; + } else if ( + parsedPkg['peerDependencies'] != null && + parsedPkg['peerDependencies'][depNpmName] != null) { + version = parsedPkg['peerDependencies'][depNpmName]; + } else if ( + parsedPkg['devDependencies'] != null && + parsedPkg['devDependencies'][depNpmName] != null) { + version = parsedPkg['devDependencies'][depNpmName]; + } + if (version == null) { + throw new Error(`No dependency found for ${dep}.`); + } - pkg = `${pkg}`.replace( - new RegExp(`"${depNpmName}": "${version}"`, 'g'), - `"${depNpmName}": "${versionLatest}"`); + let relaxedVersionPrefix = ''; + if (version.startsWith('~') || version.startsWith('^')) { + relaxedVersionPrefix = version.substr(0, 1); } - } + const versionLatest = relaxedVersionPrefix + newVersion; + pkg = `${pkg}`.replace( + new RegExp(`"${depNpmName}": "${version}"`, 'g'), + `"${depNpmName}": "${versionLatest}"`); + } return pkg; } @@ -403,3 +405,25 @@ export function createPR( $(`hub pull-request -b ${releaseBranch} -m "${message}" -l INTERNAL -o`); console.log(); } + +// Computes the default updated version (does a patch version update). +export function getPatchUpdateVersion(version: string): string { + const versionSplit = version.split('.'); + + // For alpha or beta version string (e.g. "0.0.1-alpha.5"), increase the + // number after alpha/beta. + if (versionSplit[2].includes('alpha') || versionSplit[2].includes('beta')) { + return [ + versionSplit[0], versionSplit[1], versionSplit[2], +versionSplit[3] + 1 + ].join('.'); + } + + return [versionSplit[0], versionSplit[1], +versionSplit[2] + 1].join('.'); +} + +// Computes the default updated version (does a minor version update). +export function getMinorUpdateVersion(version: string): string { + const versionSplit = version.split('.'); + + return [versionSplit[0], +versionSplit[1] + 1, '0'].join('.'); +} diff --git a/scripts/release.ts b/scripts/release.ts index 7d2086d9177..b9ea012ccf7 100644 --- a/scripts/release.ts +++ b/scripts/release.ts @@ -30,7 +30,7 @@ import * as argparse from 'argparse'; import chalk from 'chalk'; import * as fs from 'fs'; import * as shell from 'shelljs'; -import {RELEASE_UNITS, WEBSITE_RELEASE_UNIT, TMP_DIR, $, question, printReleaseUnit, printPhase, makeReleaseDir, updateDependency, prepareReleaseBuild, createPR} from './release-util'; +import {RELEASE_UNITS, WEBSITE_RELEASE_UNIT, TMP_DIR, $, question, printReleaseUnit, printPhase, makeReleaseDir, updateDependency, prepareReleaseBuild, createPR, getPatchUpdateVersion} from './release-util'; import {releaseWebsite} from './release-website'; const parser = new argparse.ArgumentParser(); @@ -40,21 +40,6 @@ parser.addArgument('--git-protocol', { help: 'Use the git protocal rather than the http protocol when cloning repos.' }); -// Computes the default updated version (does a patch version update). -function getPatchUpdateVersion(version: string): string { - const versionSplit = version.split('.'); - - // For alpha or beta version string (e.g. "0.0.1-alpha.5"), increase the - // number after alpha/beta. - if (versionSplit[2].includes('alpha') || versionSplit[2].includes('beta')) { - return [ - versionSplit[0], versionSplit[1], versionSplit[2], +versionSplit[3] + 1 - ].join('.'); - } - - return [versionSplit[0], versionSplit[1], +versionSplit[2] + 1].join('.'); -} - async function main() { const args = parser.parseArgs(); diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index 578cc4698f4..b9152d87ad8 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -3,5 +3,6 @@ "compilerOptions": { "module": "commonjs", "target": "es5", + "downlevelIteration": true } } From a74ca21134a40d29a9597b86ed5925f2639db84e Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 9 Mar 2022 14:29:55 -0800 Subject: [PATCH 4/5] Make alpha packages publish using the tfjs branch --- scripts/publish-npm.ts | 13 ++++++++++--- scripts/release-util.ts | 2 +- scripts/release.ts | 12 ++++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/scripts/publish-npm.ts b/scripts/publish-npm.ts index 7e982551738..f85871fade2 100755 --- a/scripts/publish-npm.ts +++ b/scripts/publish-npm.ts @@ -25,7 +25,7 @@ import * as argparse from 'argparse'; import chalk from 'chalk'; import * as shell from 'shelljs'; -import {RELEASE_UNITS, question, $, printReleaseUnit, printPhase, getReleaseBranch, checkoutReleaseBranch} from './release-util'; +import {RELEASE_UNITS, question, $, printReleaseUnit, printPhase, getReleaseBranch, checkoutReleaseBranch, ALPHA_RELEASE_UNIT, TFJS_RELEASE_UNIT} from './release-util'; import * as fs from 'fs'; const TMP_DIR = '/tmp/tfjs-publish'; @@ -56,7 +56,8 @@ async function main() { console.log(chalk.blue(`Using release unit ${releaseUnitInt}`)); console.log(); - const {name, phases} = RELEASE_UNITS[releaseUnitInt]; + const releaseUnit = RELEASE_UNITS[releaseUnitInt]; + const {name, phases} = releaseUnit; phases.forEach((_, i) => printPhase(phases, i)); console.log(); @@ -70,7 +71,13 @@ async function main() { console.log(chalk.blue(`Using phase ${phaseInt}`)); console.log(); - let releaseBranch = await getReleaseBranch(name); + let releaseBranch: string; + if (releaseUnit === ALPHA_RELEASE_UNIT) { + // Alpha release unit is published with the tfjs release unit. + releaseBranch = await getReleaseBranch(TFJS_RELEASE_UNIT.name); + } else { + releaseBranch = await getReleaseBranch(name); + } console.log(); checkoutReleaseBranch(releaseBranch, args.git_protocol, TMP_DIR); diff --git a/scripts/release-util.ts b/scripts/release-util.ts index 9baa3dfda3e..d2d2d19fbc8 100755 --- a/scripts/release-util.ts +++ b/scripts/release-util.ts @@ -146,7 +146,7 @@ export const TFJS_RELEASE_UNIT: ReleaseUnit = { // The release script will ask for a new version for each phase, and it will // replace 'link' dependencies with the new monorepo version. export const ALPHA_RELEASE_UNIT: ReleaseUnit = { - name: 'alpha monorepo packages', + name: 'alpha-monorepo-packages', phases: [WEBGPU_PHASE], }; diff --git a/scripts/release.ts b/scripts/release.ts index b9ea012ccf7..946469bd96e 100644 --- a/scripts/release.ts +++ b/scripts/release.ts @@ -30,7 +30,7 @@ import * as argparse from 'argparse'; import chalk from 'chalk'; import * as fs from 'fs'; import * as shell from 'shelljs'; -import {RELEASE_UNITS, WEBSITE_RELEASE_UNIT, TMP_DIR, $, question, printReleaseUnit, printPhase, makeReleaseDir, updateDependency, prepareReleaseBuild, createPR, getPatchUpdateVersion} from './release-util'; +import {RELEASE_UNITS, WEBSITE_RELEASE_UNIT, TMP_DIR, $, question, printReleaseUnit, printPhase, makeReleaseDir, updateDependency, prepareReleaseBuild, createPR, getPatchUpdateVersion, ALPHA_RELEASE_UNIT} from './release-util'; import {releaseWebsite} from './release-website'; const parser = new argparse.ArgumentParser(); @@ -43,20 +43,24 @@ parser.addArgument('--git-protocol', { async function main() { const args = parser.parseArgs(); - RELEASE_UNITS.forEach((_, i) => printReleaseUnit(i)); + // The alpha release unit is released with the monorepo and should not be + // released by this script. Packages in the alpha release unit need their + // package.json dependencies rewritten. + const releaseUnits = RELEASE_UNITS.filter(r => r !== ALPHA_RELEASE_UNIT); + releaseUnits.forEach((_, i) => printReleaseUnit(i)); console.log(); const releaseUnitStr = await question('Which release unit (leave empty for 0): '); const releaseUnitInt = +releaseUnitStr; - if (releaseUnitInt < 0 || releaseUnitInt >= RELEASE_UNITS.length) { + if (releaseUnitInt < 0 || releaseUnitInt >= releaseUnits.length) { console.log(chalk.red(`Invalid release unit: ${releaseUnitStr}`)); process.exit(1); } console.log(chalk.blue(`Using release unit ${releaseUnitInt}`)); console.log(); - const releaseUnit = RELEASE_UNITS[releaseUnitInt]; + const releaseUnit = releaseUnits[releaseUnitInt]; const {name, phases} = releaseUnit; phases.forEach((_, i) => printPhase(phases, i)); From e5010988962ce2c6cd7dbacd218af097f546c800 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 9 Mar 2022 14:59:31 -0800 Subject: [PATCH 5/5] Update release-util.ts --- scripts/release-util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release-util.ts b/scripts/release-util.ts index d2d2d19fbc8..ecc031d82a3 100755 --- a/scripts/release-util.ts +++ b/scripts/release-util.ts @@ -140,7 +140,7 @@ export const TFJS_RELEASE_UNIT: ReleaseUnit = { // TODO(mattsoulanille): Move WEBGPU_PHASE to TFJS_RELEASE_UNIT when webgpu // is out of alpha. -// Alpha packages that use monorepo dependencies at the latest version but are +// Alpha packages use monorepo dependencies at the latest version but are // not yet released at the same version number as the monorepo packages. // Use this for packages that will be a part of the monorepo in the future. // The release script will ask for a new version for each phase, and it will