diff --git a/.buildkite/bootstrap.sh b/.buildkite/bootstrap.sh index c836c7a4c6..5fe05c9b74 100755 --- a/.buildkite/bootstrap.sh +++ b/.buildkite/bootstrap.sh @@ -12,7 +12,7 @@ fi # Get the change set git diff-tree --no-commit-id --name-only -r $SHA1 $SHA2 | grep -v "\.md$" > changes.txt -CHANGES=$(jazelle changes ./changes.txt) +CHANGES=$(jazelle changes --format dirs ./changes.txt) for DIR in $CHANGES ; do ( PROJECT=$(basename "$DIR"); diff --git a/jazelle/README.md b/jazelle/README.md index 986c67009b..63cc93643a 100644 --- a/jazelle/README.md +++ b/jazelle/README.md @@ -335,6 +335,7 @@ If you get into a bad state, here are some things you can try: - [`jazelle dedupe`](#jazelle-dedupe) - [`jazelle purge`](#jazelle-purge) - [`jazelle check`](#jazelle-check) +- [`jazelle align`](#jazelle-align) - [`jazelle chunk`](#jazelle-chunk) - [`jazelle changes`](#jazelle-changes) - [`jazelle plan`](#jazelle-plan) @@ -461,6 +462,14 @@ Shows a report of out-of-sync top level dependencies across projects } ``` +### `jazelle align` + +Align a project's dependency versions to respect the version policy, if there is one + +`jazelle align --cwd [cwd]` + +- `--cwd` - Project folder (absolute or relative to shell `cwd`). Defaults to `process.cwd()` + ### `jazelle chunk` Prints a glob pattern representing a chunk of files matching a given list of glob patterns. Useful for splitting tests across multiple CI jobs. @@ -673,6 +682,7 @@ If you want commands to display colorized output, run their respective NPM scrip - [dedupe](#dedupe) - [purge](#purge) - [check](#check) +- [align](#align) - [chunk](#chunk) - [changes](#changes) - [plan](#plan) @@ -820,6 +830,15 @@ let check: ({root: string, projects: Array, versionPolicy: VersionPolicy - `root` - Monorepo root folder (absolute path) +### `align` + +Align a project's dependency versions to respect the version policy, if there is one + +`let align: ({root: string, cwd: string}) => Promise` + +- `root` - Monorepo root folder (absolute path) +- `cwd` - Project folder (absolute path) + ### `chunk` Returns a glob pattern representing a chunk of files matching a given list of glob patterns. Useful for splitting tests across multiple CI jobs. diff --git a/jazelle/bin/cluster.js b/jazelle/bin/cluster.js index 67c9d3908e..f3a3701bff 100644 --- a/jazelle/bin/cluster.js +++ b/jazelle/bin/cluster.js @@ -48,7 +48,7 @@ async function runMaster() { for (const data of payload) { if (data.length > 0) { const requiredCores = Math.min(availableCores, data.length); - const workers = [...Array(requiredCores)].map(() => fork()); + const workers = [...Array(requiredCores)].map(() => fork(process.env)); await install({ root, @@ -79,7 +79,7 @@ async function runMaster() { const command = data.shift(); const log = `${tmpdir()}/${Math.random() * 1e17}`; - if (worker.state === 'dead') worker = fork(); + if (worker.state === 'dead') worker = fork(process.env); worker.send({command, log}); worker.once('exit', async () => { diff --git a/jazelle/commands/add.js b/jazelle/commands/add.js index cb78c6efdf..b56f1d86eb 100644 --- a/jazelle/commands/add.js +++ b/jazelle/commands/add.js @@ -3,8 +3,7 @@ const {resolve} = require('path'); const {assertProjectDir} = require('../utils/assert-project-dir.js'); const {getManifest} = require('../utils/get-manifest.js'); const {getLocalDependencies} = require('../utils/get-local-dependencies.js'); -const {node, yarn} = require('../utils/binary-paths.js'); -const {exec, read, write} = require('../utils/node-helpers.js'); +const {read, write} = require('../utils/node-helpers.js'); const {findLocalDependency} = require('../utils/find-local-dependency.js'); const {add: addDep} = require('../utils/lockfile.js'); const {install} = require('./install.js'); @@ -12,6 +11,7 @@ const {install} = require('./install.js'); /* adding local dep should: - add it to the project's package.json, pointing to the exact local version +- update the BUILD.bazel file `deps` field - not add it to the project's yarn.lock */ @@ -19,54 +19,57 @@ adding local dep should: export type AddArgs = { root: string, cwd: string, - name: string, + args: Array, version?: string, dev?: boolean, }; export type Add = (AddArgs) => Promise; */ -const add /*: Add */ = async ({ - root, - cwd, - name: nameWithVersion, - dev = false, -}) => { +const add /*: Add */ = async ({root, cwd, args, dev = false}) => { await assertProjectDir({dir: cwd}); - let [, name, version] = nameWithVersion.match(/(@?[^@]*)@?(.*)/) || []; + const type = dev ? 'devDependencies' : 'dependencies'; - const local = await findLocalDependency({root, name}); - if (local) { - if (version && version !== local.meta.version) { - throw new Error(`You must use version ${local.meta.version}`); - } + // group by whether the dep is local (listed in manifest.json) or external (from registry) + const locals = []; + const externals = []; + for (const arg of args) { + let [, name, version] = arg.match(/(@?[^@]*)@?(.*)/) || []; + const local = await findLocalDependency({root, name}); + if (local) locals.push({local, name}); + else externals.push({name, range: version, type}); + } + + // add local deps + if (locals.length > 0) { const meta = JSON.parse(await read(`${cwd}/package.json`, 'utf8')); if (!meta[type]) meta[type] = {}; - const types = [ - 'dependencies', - 'devDependencies', - 'peerDependencies', - 'optionalDependencies', - ]; - for (const t of types) { - if (meta[t] && meta[t][name]) { - meta[t][name] = local.meta.version; + + for (const {local, name} of locals) { + // update existing entries + const types = [ + 'dependencies', + 'devDependencies', + 'peerDependencies', + 'optionalDependencies', + 'resolutions', + ]; + for (const t of types) { + if (meta[t] && meta[t][name]) { + meta[t][name] = local.meta.version; + } } + meta[type][name] = local.meta.version; } - meta[type][name] = local.meta.version; await write( `${cwd}/package.json`, `${JSON.stringify(meta, null, 2)}\n`, 'utf8' ); - } else { - // adding does not dedupe transitives, since consumers will rarely want to check if they introduced regressions in unrelated projects - if (!version) { - version = JSON.parse( - await exec(`${node} ${yarn} info ${name} version --json 2>/dev/null`) - ).data; - } - const additions = [{name, range: version, type}]; + } + + // add external deps + if (externals.length > 0) { const {projects} = await getManifest({root}); const deps = await getLocalDependencies({ dirs: projects.map(dir => `${root}/${dir}`), @@ -75,11 +78,12 @@ const add /*: Add */ = async ({ const tmp = `${root}/third_party/jazelle/temp/yarn-utilities-tmp`; await addDep({ roots: [cwd], - additions, + additions: externals, ignore: deps.map(d => d.meta.name), tmp, }); } + await install({root, cwd}); }; diff --git a/jazelle/commands/align.js b/jazelle/commands/align.js new file mode 100644 index 0000000000..25da454d6c --- /dev/null +++ b/jazelle/commands/align.js @@ -0,0 +1,63 @@ +// @flow +const {getManifest} = require('../utils/get-manifest.js'); +const {getAllDependencies} = require('../utils/get-all-dependencies.js'); +const {read, write} = require('../utils/node-helpers.js'); +const {install} = require('./install.js'); + +/*:: +type AlignArgs = { + root: string, + cwd: string, +} +type Align = (AlignArgs) => Promise +*/ +const align /*: Align */ = async ({root, cwd}) => { + const {projects, versionPolicy} = /*:: await */ await getManifest({root}); + if (versionPolicy) { + const deps = await getAllDependencies({root, projects}); + const meta = JSON.parse(await read(`${cwd}/package.json`, 'utf8')); + const others = deps.filter(dep => dep.meta.name !== meta.name); + const types = ['dependencies', 'devDependencies', 'resolutions']; + let changed = false; + for (const type of types) { + if (meta[type]) { + for (const name in meta[type]) { + if (shouldSyncVersions({versionPolicy, name})) { + const version = getVersion({name, deps: others}); + if (version !== null) { + meta[type][name] = version; + changed = true; + } + } + } + } + } + if (changed) { + const modified = JSON.stringify(meta, null, 2) + '\n'; + await write(`${cwd}/package.json`, modified, 'utf8'); + } + } + await install({root, cwd}); +}; + +const shouldSyncVersions = ({versionPolicy, name}) => { + const {lockstep = false, exceptions = []} = versionPolicy; + return ( + (lockstep && !exceptions.includes(name)) || + (!lockstep && exceptions.includes(name)) + ); +}; + +const getVersion = ({name, deps}) => { + const types = ['dependencies', 'devDependencies', 'resolutions']; + for (const {meta} of deps) { + for (const type of types) { + for (const key in meta[type]) { + if (name === key) return meta[type][key]; + } + } + } + return null; +}; + +module.exports = {align}; diff --git a/jazelle/commands/outdated.js b/jazelle/commands/outdated.js new file mode 100644 index 0000000000..d1659dfd7c --- /dev/null +++ b/jazelle/commands/outdated.js @@ -0,0 +1,55 @@ +// @flow +const {getManifest} = require('../utils/get-manifest.js'); +const {getAllDependencies} = require('../utils/get-all-dependencies.js'); +const {exec} = require('../utils/node-helpers.js'); +const {node, yarn} = require('../utils/binary-paths.js'); +const {minVersion, gt} = require('semver'); + +/*:: +type OutdatedArgs = { + root: string, +}; +type Outdated = (OutdatedArgs) => Promise +*/ +const outdated /*: Outdated */ = async ({root}) => { + const {projects} = await getManifest({root}); + const locals = await getAllDependencies({root, projects}); + const map = {}; + const types = ['dependencies', 'devDependencies']; + for (const local of locals) { + for (const type of types) { + if (local.meta[type]) { + for (const name in local.meta[type]) { + if (!map[name]) map[name] = new Set(); + map[name].add(local.meta[type][name]); + } + } + } + } + // report local discrepancies + for (const name in map) { + const local = locals.find(local => local.meta.name === name); + if (local) { + const {version} = local.meta; + for (const range of map[name]) { + if (version !== range) console.log(name, range, version); + } + } + } + // then report registry discrepancies + for (const name in map) { + const local = locals.find(local => local.meta.name === name); + if (!local) { + const query = `${node} ${yarn} info ${name} version --json 2>/dev/null`; + const registryVersion = await exec(query); + if (registryVersion) { + const latest = JSON.parse(registryVersion).data; + for (const range of map[name]) { + if (gt(latest, minVersion(range))) console.log(name, range, latest); + } + } + } + } +}; + +module.exports = {outdated}; diff --git a/jazelle/index.js b/jazelle/index.js index c249bf890a..56a60928fa 100644 --- a/jazelle/index.js +++ b/jazelle/index.js @@ -12,6 +12,8 @@ const {upgrade} = require('./commands/upgrade.js'); const {dedupe} = require('./commands/dedupe.js'); const {purge} = require('./commands/purge.js'); const {check} = require('./commands/check.js'); +const {outdated} = require('./commands/outdated.js'); +const {align} = require('./commands/align.js'); const {chunk} = require('./commands/chunk.js'); const {changes} = require('./commands/changes.js'); const {plan} = require('./commands/plan.js'); @@ -77,11 +79,11 @@ const runCLI /*: RunCLI */ = async argv => { [name] Package to add at a specific version. ie., foo@1.2.3 --dev Whether to install as devDependency --cwd [cwd] Project directory to use`, - async ({cwd, name, dev}) => + async ({cwd, dev}) => add({ root: await rootOf(args), cwd, - name: name || dev, // if dev is passed before name, resolve to correct value + args: rest.filter(arg => arg != '--dev'), // if dev is passed before name, resolve to correct value dev: Boolean(dev), // FIXME all args can technically be boolean, but we don't want Flow complaining about it everywhere }), ], @@ -113,6 +115,16 @@ const runCLI /*: RunCLI */ = async argv => { async ({json}) => check({root: await rootOf(args), json: Boolean(json)}), ], + outdated: [ + `Displays deps whose version is behind the latest version`, + async () => outdated({root: await rootOf(args)}), + ], + align: [ + `Align a project's dependency versions to respect the version policy, if there is one + + --cwd [cwd] Project directory to use`, + async ({cwd}) => await align({root: await rootOf(args), cwd}), + ], chunk: [ `Print a glob pattern representing a chunk of a set of files @@ -264,6 +276,7 @@ module.exports = { dedupe, purge, check: reportMismatchedTopLevelDeps, + align, chunk: getChunkPattern, changes: findChangedTargets, plan: getTestGroups, diff --git a/jazelle/rules/execute-command.js b/jazelle/rules/execute-command.js index ead53088e0..2f845d8701 100644 --- a/jazelle/rules/execute-command.js +++ b/jazelle/rules/execute-command.js @@ -4,7 +4,8 @@ const {execSync: exec} = require('child_process'); const {dirname, basename} = require('path'); const root = process.cwd(); -const [node, , main, bin, command, dist, out, ...args] = process.argv; +const [node, , main, bin, command, distPaths, out, ...args] = process.argv; +const dists = distPaths.split('|'); const files = exec(`find . -name output.tgz`, {cwd: bin, encoding: 'utf8'}) .split('\n') @@ -22,9 +23,12 @@ files.map(f => { const {scripts = {}} = JSON.parse(read(`${main}/package.json`, 'utf8')); if (out) { - exec(`mkdir -p "${dist}"`, {cwd: main}); + for (const dist of dists) { + exec(`mkdir -p "${dist}"`, {cwd: main}); + } runCommands(); - exec(`tar czf "${out}" "${dist}"`, {cwd: main}); + const dirs = dists.map(dist => `"${dist}"`).join(' '); + exec(`tar czf "${out}" ${dirs}`, {cwd: main}); } else { try { runCommands(); diff --git a/jazelle/rules/web-monorepo.bzl b/jazelle/rules/web-monorepo.bzl index 82b166d10c..36963e16ea 100644 --- a/jazelle/rules/web-monorepo.bzl +++ b/jazelle/rules/web-monorepo.bzl @@ -75,7 +75,7 @@ def _web_binary_impl(ctx): node = ctx.files._node[0].path, srcdir = ctx.build_file_path, command = ctx.attr.build, - dist = ctx.attr.dist[0], + dist = "|".join(ctx.attr.dist), output = build_ouput.path, bindir = ctx.bin_dir.path, build = ctx.files._script[0].path, diff --git a/jazelle/tests/fixtures/find-changed-targets/bazel/changes.txt b/jazelle/tests/fixtures/find-changed-targets/bazel/changes.txt index 103eaea598..03ea6a126c 100644 --- a/jazelle/tests/fixtures/find-changed-targets/bazel/changes.txt +++ b/jazelle/tests/fixtures/find-changed-targets/bazel/changes.txt @@ -1 +1,2 @@ -b/package.json \ No newline at end of file +b/package.json +non-jazelle/BUILD.bazel \ No newline at end of file diff --git a/jazelle/tests/fixtures/find-changed-targets/bazel/non-jazelle/BUILD.bazel b/jazelle/tests/fixtures/find-changed-targets/bazel/non-jazelle/BUILD.bazel new file mode 100644 index 0000000000..ac2d687464 --- /dev/null +++ b/jazelle/tests/fixtures/find-changed-targets/bazel/non-jazelle/BUILD.bazel @@ -0,0 +1,52 @@ +package(default_visibility = ["//visibility:public"]) + +load("@jazelle//:build-rules.bzl", "web_library", "web_binary", "web_executable", "web_test", "flow_test") + +web_library( + name = "library", + deps = [ + "//:node_modules", + ], + srcs = glob(["**"], exclude = ["dist/**"]), +) + +web_binary( + name = "non-jazelle", + build = "build", + command = "start", + deps = [ + "//non-jazelle:library", + ], + dist = ["dist"], +) + +web_executable( + name = "dev", + command = "dev", + deps = [ + "//non-jazelle:library", + ], +) + +web_test( + name = "test", + command = "test", + deps = [ + "//non-jazelle:library", + ], +) + +web_test( + name = "lint", + command = "lint", + deps = [ + "//non-jazelle:library", + ], +) + +flow_test( + name = "flow", + deps = [ + "//non-jazelle:library", + ], +) \ No newline at end of file diff --git a/jazelle/tests/index.js b/jazelle/tests/index.js index 47d7adba80..adb254e144 100644 --- a/jazelle/tests/index.js +++ b/jazelle/tests/index.js @@ -80,7 +80,6 @@ async function runTests() { await exec(`mkdir -p ${__dirname}/tmp`); await Promise.all([ - t(testInstallAddUpgradeRemove), t(testCi), t(testDedupe), t(testUpgrade), @@ -123,6 +122,7 @@ async function runTests() { t(testPopulateGraph), ]); // run separately to avoid CI error + await t(testInstallAddUpgradeRemove); await t(testCommand); await t(testYarnCommand); await t(testBazelCommand); @@ -160,7 +160,7 @@ async function testInstallAddUpgradeRemove() { await add({ root: `${__dirname}/tmp/commands`, cwd: `${__dirname}/tmp/commands/a`, - name: 'c', + args: ['c'], }); assert(await exists(`${__dirname}/tmp/commands/node_modules/c`)); assert((await read(buildFile, 'utf8')).includes('//c:c')); @@ -169,7 +169,7 @@ async function testInstallAddUpgradeRemove() { await add({ root: `${__dirname}/tmp/commands`, cwd: `${__dirname}/tmp/commands/a`, - name: 'has@1.0.3', + args: ['has@1.0.3'], }); assert(JSON.parse(await read(meta, 'utf8')).dependencies.has); assert(await exists(`${__dirname}/tmp/commands/node_modules/has`)); diff --git a/jazelle/utils/find-changed-targets.js b/jazelle/utils/find-changed-targets.js index d8fd29f355..5297058006 100644 --- a/jazelle/utils/find-changed-targets.js +++ b/jazelle/utils/find-changed-targets.js @@ -46,7 +46,7 @@ const findChangedBazelTargets = async ({root, files}) => { const targets = result.split('\n').filter(Boolean); return {workspace, targets}; } else { - const projects = await batch(root, lines, async file => { + const queried = await batch(root, lines, async file => { const find = `${bazel} query "${file}"`; const result = await exec(find, opts).catch(async e => { // if file doesn't exist, find which package it would've belong to, and find another file in the same package @@ -66,10 +66,14 @@ const findChangedBazelTargets = async ({root, files}) => { const project = await exec(cmd, opts); return project; }); - const targets = await batch(root, projects, async project => { + const unfiltered = await batch(root, queried, async project => { const cmd = `${bazel} query 'let graph = kind(".*_test rule", rdeps("...", "${project}")) in $graph except filter("node_modules", $graph)'`; return exec(cmd, opts); }); + const targets = unfiltered.filter(target => { + const path = target.replace(/\/\/(.+?):.+/, '$1'); + return projects.includes(path); + }); return {workspace, targets}; } } else { diff --git a/jazelle/utils/generate-bazel-build-rules.js b/jazelle/utils/generate-bazel-build-rules.js index 9125b59703..84a107950e 100644 --- a/jazelle/utils/generate-bazel-build-rules.js +++ b/jazelle/utils/generate-bazel-build-rules.js @@ -65,7 +65,10 @@ const generateBazelBuildRules /*: GenerateBazelBuildRules */ = async ({ dependencies .map(d => `"${d}"`) .forEach(dependency => { - if (!items.includes(dependency)) { + // only add if no related target exists + const [path] = dependency.split(':'); + const paths = items.map(item => item.split(':').shift()); + if (!paths.includes(path)) { code = addCallArgItem( code, dependencySyncRule, diff --git a/jazelle/utils/install-deps.js b/jazelle/utils/install-deps.js index 1e34cd3ca4..425906ff34 100644 --- a/jazelle/utils/install-deps.js +++ b/jazelle/utils/install-deps.js @@ -119,7 +119,7 @@ const generateNodeModules = async bin => { const args = [ yarn, 'install', - '--frozen-lockfile', + '--pure-lockfile', '--non-interactive', '--ignore-optional', ];