From 1a47fe1be887043b4056305b4bab621a7164a780 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Wed, 20 Nov 2019 12:34:16 +0100 Subject: [PATCH 1/2] build: rework package-builder script to node script Reworks the package-builder script from a Bash script to a NodeJS script. This makes it work better in all platforms, and makes the script more maintainable. Best example is how easy it can be integrated into the release tool. Framework did the same recently. --- .circleci/config.yml | 2 +- package.json | 2 +- scripts/build-packages-dist.js | 122 +++++++++++++++++++++++++++++++ scripts/build-packages-dist.sh | 81 -------------------- tools/release/publish-release.ts | 17 ++--- 5 files changed, 129 insertions(+), 95 deletions(-) create mode 100644 scripts/build-packages-dist.js delete mode 100755 scripts/build-packages-dist.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index e40d3cfd4d52..d5260de92469 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -297,7 +297,7 @@ jobs: - *yarn_install - *setup_bazel_binary - - run: ./scripts/build-packages-dist.sh + - run: yarn build - run: yarn check-release-output # TODO(devversion): replace this with bazel tests that run Madge. This is diff --git a/package.json b/package.json index 562cfb4b7f94..b8eedc205155 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ }, "scripts": { "postinstall": "node tools/bazel/postinstall-patches.js && ngcc --properties main --create-ivy-entry-points", - "build": "bash ./scripts/build-packages-dist.sh", + "build": "node ./scripts/build-packages-dist.js", "bazel:buildifier": "find . -type f \\( -name \"*.bzl\" -or -name WORKSPACE -or -name BUILD -or -name BUILD.bazel \\) ! -path \"*/node_modules/*\" | xargs buildifier -v --warnings=attr-cfg,attr-license,attr-non-empty,attr-output-default,attr-single-file,constant-glob,ctx-args,depset-iteration,depset-union,dict-concatenation,duplicated-name,filetype,git-repository,http-archive,integer-division,load,load-on-top,native-build,native-package,output-group,package-name,package-on-top,redefined-variable,repository-name,same-origin-load,string-iteration,unused-variable,unsorted-dict-items,out-of-order-load", "bazel:format-lint": "yarn -s bazel:buildifier --lint=warn --mode=check", "dev-app": "ibazel run //src/dev-app:devserver", diff --git a/scripts/build-packages-dist.js b/scripts/build-packages-dist.js new file mode 100644 index 000000000000..e5f8d5c44de3 --- /dev/null +++ b/scripts/build-packages-dist.js @@ -0,0 +1,122 @@ +/** + * Script that builds the release output of all packages which have the "release-package + * bazel tag set. The script builds all those packages and copies the release output to the + * distribution folder within the project. + */ + +const {execSync} = require('child_process'); +const {join} = require('path'); +const {chmod, cp, mkdir, rm, set, test} = require('shelljs'); + +// ShellJS should exit if a command fails. +set('-e'); + +/** Name of the Bazel tag that will be used to find release package targets. */ +const releaseTargetTag = 'release-package'; + +/** Path to the project directory. */ +const projectDir = join(__dirname, '../'); + +/** Command that runs Bazel. */ +const bazelCmd = process.env.BAZEL_COMMAND || `yarn -s bazel`; + +/** Command that queries Bazel for all release package targets. */ +const queryPackagesCmd = + `${bazelCmd} query --output=label "attr('tags', '\\[.*${releaseTargetTag}.*\\]', //src/...) ` + + `intersect kind('.*_package', //src/...)"`; + +// Export the methods for building the release packages. These +// can be consumed by the release tool. +exports.buildReleasePackages = buildReleasePackages; +exports.defaultBuildReleasePackages = defaultBuildReleasePackages; + +if (module === require.main) { + defaultBuildReleasePackages(); +} + +/** + * Builds the release packages with the default compile mode and + * output directory. + */ +function defaultBuildReleasePackages() { + buildReleasePackages('legacy', join(projectDir, 'dist/releases')); +} + +/** + * Builds the release packages with the given compile mode and copies + * the package output into the given directory. + */ +function buildReleasePackages(compileMode, distPath) { + console.log('######################################'); + console.log(' Building release packages...'); + console.log(` Compile mode: ${compileMode}`); + console.log('######################################'); + + // List of targets to build. e.g. "src/cdk:npm_package", or "src/material:npm_package". + const targets = exec(queryPackagesCmd, true).split(/\r?\n/); + const packageNames = getPackageNamesOfTargets(targets); + const bazelBinPath = exec(`${bazelCmd} info bazel-bin`, true); + const getOutputPath = pkgName => join(bazelBinPath, 'src', pkgName, 'npm_package'); + + // Walk through each release package and clear previous "npm_package" outputs. This is + // a workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1219. We need to + // do this to ensure that the version placeholders are properly populated. + packageNames.forEach(pkgName => { + const outputPath = getOutputPath(pkgName); + if (test('-d', outputPath)) { + chmod('-R', 'u+w', outputPath); + rm('-rf', outputPath); + } + }); + + // Build with "--config=release" so that Bazel runs the workspace stamping script. The + // stamping script ensures that the version placeholder is populated in the release output. + exec(`${bazelCmd} build --config=release --define=compile=${compileMode} ${targets.join(' ')}`); + + // Delete the distribution directory so that the output is guaranteed to be clean. Re-create + // the empty directory so that we can copy the release packages into it later. + rm('-rf', distPath); + mkdir('-p', distPath); + + // Copy the package output into the specified distribution folder. + packageNames.forEach(pkgName => { + const outputPath = getOutputPath(pkgName); + const targetFolder = join(distPath, pkgName); + console.log(`> Copying package output to "${targetFolder}"`); + cp('-R', outputPath, targetFolder); + chmod('-R', 'u+w', targetFolder); + }); +} + +/** + * Gets the package names of the specified Bazel targets. + * e.g. //src/material:npm_package -> material + */ +function getPackageNamesOfTargets(targets) { + return targets.map(targetName => { + const matches = targetName.match(/\/\/src\/(.*):npm_package/); + if (matches === null) { + throw Error(`Found Bazel target with "${releaseTargetTag}" tag, but could not ` + + `determine release output name: ${targetName}`); + } + return matches[1]; + }); +} + +/** + * Executes the given command in the project directory. + * @param {string} command The command to run + * @param {boolean=} captureStdout Whether the stdout should be captured and + * returned. + */ +function exec(command, captureStdout) { + const stdout = execSync(command, { + cwd: projectDir, + stdio: ['inherit', captureStdout ? 'pipe' : 'inherit', 'inherit'], + }); + + if (captureStdout) { + process.stdout.write(stdout); + return stdout.toString().trim(); + } +} diff --git a/scripts/build-packages-dist.sh b/scripts/build-packages-dist.sh deleted file mode 100755 index 95040a78c1ab..000000000000 --- a/scripts/build-packages-dist.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# Script that builds the release output of all packages which have the "release-package" -# bazel tag set. The script builds all those packages and copies the release output to a -# folder within the project. - -set -u -e -o pipefail - -# Go to project directory. -cd $(dirname ${0})/.. - -# Either "legacy" (view engine) or "aot" (ivy) -compile_mode=${1:-"legacy"} - -# Path to the bazel binary. By default uses "bazel" from the node modules but developers -# can overwrite the binary though an environment variable. Also by default if we run Bazel -# from the node modules, we don't want to access bazel through Yarn and NodeJS because it -# could mean that the Bazel child process only has access to limited memory. -bazel=${BAZEL_BIN_PATH:-$(yarn bin bazel)} - -echo "######################################" -echo " building release packages" -echo " mode: ${compile_mode}" -echo "######################################" -echo "" - -# Path to the output directory into which we copy the npm packages. -dest_path="dist/releases" - -# Path to the bazel-bin directory. -bazel_bin_path=$(${bazel} info bazel-bin) - -# List of targets that need to be built, e.g. //src/lib, //src/cdk, etc. Note we need to remove all -# carriage returns because Bazel prints these on Windows. This breaks the Bash array parsing. -targets=$(${bazel} query --output=label 'attr("tags", "\[.*release-package.*\]", //src/...)' \ - 'intersect kind(".*_package", //src/...)' | tr -d "\r") - -# Extracts the package name from the Bazel target names. -# e.g. `src/material:npm_package` will result in "material". -dirs=`echo "$targets" | sed -e 's/\/\/src\/\(.*\):npm_package/\1/'` - -# Walk through each release package and clear previous "npm_package" outputs. This is -# a workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1219. We need to -# do this to ensure that the version placeholders are properly populated. -for pkg in ${dirs}; do - pkg_dir="${bazel_bin_path}/src/${pkg}/npm_package" - if [[ -d ${pkg_dir} ]]; then - # Make all directories in the previous package output writable. Bazel by default - # makes tree artifacts and file outputs readonly. This causes permission errors - # when deleting the folder. To avoid these errors, we make all files writable. - chmod -R u+w ${pkg_dir} - rm -Rf ${pkg_dir} - fi -done - -# Walk through each release package target and build it. -for target in ${targets}; do - echo -e "Building: ${target} ...\n" - # Build with "--config=release" so that Bazel runs the workspace stamping script. The - # stamping script ensures that the version placeholder is populated in the release output. - ${bazel} build --config=release --define=compile=${compile_mode} ${target} - echo "" -done - -# Delete the distribution directory so that the output is guaranteed to be clean. Re-create -# the empty directory so that we can copy the release packages into it later. -rm -Rf ${dest_path} -mkdir -p ${dest_path} - -# Copy the package output for all built NPM packages into the dist directory. -for pkg in ${dirs}; do - pkg_dir="${bazel_bin_path}/src/${pkg}/npm_package" - target_dir="${dest_path}/${pkg}" - - if [[ -d ${pkg_dir} ]]; then - echo "> Copying package output to \"${target_dir}\".." - rm -rf ${target_dir} - cp -R ${pkg_dir} ${target_dir} - chmod -R u+w ${target_dir} - fi -done diff --git a/tools/release/publish-release.ts b/tools/release/publish-release.ts index a4b6ffa6a5a5..15c546ebcb1a 100644 --- a/tools/release/publish-release.ts +++ b/tools/release/publish-release.ts @@ -1,5 +1,4 @@ import chalk from 'chalk'; -import {spawnSync} from 'child_process'; import {readFileSync, unlinkSync} from 'fs'; import {homedir} from 'os'; import {join} from 'path'; @@ -15,6 +14,10 @@ import {releasePackages} from './release-output/release-packages'; import {CHANGELOG_FILE_NAME} from './stage-release'; import {parseVersionName, Version} from './version-name/parse-version'; +// The package builder script is not written in TypeScript and needs to +// be imported through a CommonJS import. +const {defaultBuildReleasePackages} = require('../../scripts/build-packages-dist'); + /** * Class that can be instantiated in order to create a new release. The tasks requires user * interaction/input through command line prompts. @@ -87,7 +90,7 @@ class PublishReleaseTask extends BaseReleaseTask { await this._promptStableVersionForNextTag(); } - this._buildReleasePackages(); + defaultBuildReleasePackages(); console.info(chalk.green(` ✓ Built the release output.`)); // Checks all release packages against release output validations before releasing. @@ -149,16 +152,6 @@ class PublishReleaseTask extends BaseReleaseTask { } } - /** Builds all release packages that should be published. */ - private _buildReleasePackages() { - const buildScript = join(this.projectDir, 'scripts/build-packages-dist.sh'); - - // TODO(devversion): I'd prefer disabling the output for those, but it might be only - // worth if we consider adding some terminal spinner library (like "ora"). - return spawnSync('bash', [buildScript], - {cwd: this.projectDir, stdio: 'inherit', shell: true}).status === 0; - } - /** * Prompts the user whether they are sure that the current stable version should be * released to the "next" NPM dist-tag. From 7d8a6fc70228fa14dd18f102280cee7eb0ec430b Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Wed, 20 Nov 2019 18:53:21 +0100 Subject: [PATCH 2/2] fixup! build: rework package-builder script to node script Add shebang --- scripts/build-packages-dist.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build-packages-dist.js b/scripts/build-packages-dist.js index e5f8d5c44de3..8bfd47f395a3 100644 --- a/scripts/build-packages-dist.js +++ b/scripts/build-packages-dist.js @@ -1,3 +1,5 @@ +#!/usr/bin/env node + /** * Script that builds the release output of all packages which have the "release-package * bazel tag set. The script builds all those packages and copies the release output to the