From 9a5d9d91adfe0dd649e8dab660256864661c48bc Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Tue, 3 Jan 2023 19:45:01 -0500 Subject: [PATCH] build: use bazel to perform release builds When performing a release via the dev-infra `ng-dev` tooling, the release builds for the packages that will be published are now performed using bazel. Prior to this, the release builds were performed using a custom build script that programmatically invoked TypeScript APIs. The Bazel build and discovery process for the releasable packages is performed by a script that is based on the scripts from components and framework repositories. Several small modifications were performed to match the behavior and structure of the cli repository: * Use of `packages` as the source root in the bazel query * Use of `pkg_npm` rule in the bazel query * Partial transition to native Node.js `fs` APIs instead of `shelljs` * Directory creation per package when copying output (supports multiple package scopes) * Copying of archives (tgz) for each package The snapshot and local build capabilities are not modified as part of this change but will be merged in a followup as part of a larger transition to use bazel throughout the package build process. --- .circleci/dynamic_config.yml | 3 +- .ng-dev/release.mts | 10 +- package.json | 1 + packages/angular/cli/BUILD.bazel | 1 + packages/angular/create/BUILD.bazel | 1 + packages/angular/pwa/BUILD.bazel | 1 + packages/angular_devkit/architect/BUILD.bazel | 1 + .../angular_devkit/architect/node/BUILD.bazel | 2 +- .../angular_devkit/architect_cli/BUILD.bazel | 1 + .../angular_devkit/build_angular/BUILD.bazel | 1 + .../angular_devkit/build_webpack/BUILD.bazel | 1 + packages/angular_devkit/core/BUILD.bazel | 1 + .../angular_devkit/schematics/BUILD.bazel | 1 + .../angular_devkit/schematics_cli/BUILD.bazel | 1 + packages/ngtools/webpack/BUILD.bazel | 1 + packages/schematics/angular/BUILD.bazel | 1 + scripts/build-packages-dist.mts | 168 ++++++++++++++++++ tsconfig-build.json | 3 +- yarn.lock | 13 +- 19 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 scripts/build-packages-dist.mts diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml index 5dbea5ebedca..836d01ea0e55 100644 --- a/.circleci/dynamic_config.yml +++ b/.circleci/dynamic_config.yml @@ -332,6 +332,7 @@ jobs: publish_artifacts: executor: action-executor + resource_class: medium environment: steps: - custom_attach_workspace @@ -342,7 +343,7 @@ jobs: name: Copy tarballs to folder command: | mkdir -p dist/artifacts/ - cp dist/*.tgz dist/artifacts/ + cp dist/releases/*.tgz dist/artifacts/ - store_artifacts: path: dist/artifacts/ destination: angular diff --git a/.ng-dev/release.mts b/.ng-dev/release.mts index 8e2e2333b141..ae7206c57880 100644 --- a/.ng-dev/release.mts +++ b/.ng-dev/release.mts @@ -1,8 +1,5 @@ -import '../lib/bootstrap-local.js'; - import { ReleaseConfig } from '@angular/ng-dev'; import packages from '../lib/packages.js'; -import buildPackages from '../scripts/build.js'; const npmPackages = Object.entries(packages.releasePackages).map(([name, { experimental }]) => ({ name, @@ -13,7 +10,12 @@ const npmPackages = Object.entries(packages.releasePackages).map(([name, { exper export const release: ReleaseConfig = { representativeNpmPackage: '@angular/cli', npmPackages, - buildPackages: () => buildPackages.default(), + buildPackages: async () => { + // The `performNpmReleaseBuild` function is loaded at runtime to avoid loading additional + // files and dependencies unless a build is required. + const { performNpmReleaseBuild } = await import('../scripts/build-packages-dist.mjs'); + return performNpmReleaseBuild(); + }, releaseNotes: { groupOrder: [ '@angular/cli', diff --git a/package.json b/package.json index 09a91984d86f..e638460e9752 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "@types/progress": "^2.0.3", "@types/resolve": "^1.17.1", "@types/semver": "^7.3.12", + "@types/shelljs": "^0.8.11", "@types/tar": "^6.1.2", "@types/text-table": "^0.2.1", "@types/yargs": "^17.0.8", diff --git a/packages/angular/cli/BUILD.bazel b/packages/angular/cli/BUILD.bazel index e315e76af3dc..ac63f01cbf10 100644 --- a/packages/angular/cli/BUILD.bazel +++ b/packages/angular/cli/BUILD.bazel @@ -176,6 +176,7 @@ pkg_npm( "//packages/angular_devkit/schematics:package.json", "//packages/schematics/angular:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":angular-cli", diff --git a/packages/angular/create/BUILD.bazel b/packages/angular/create/BUILD.bazel index 0b547661c54d..85973cfd9452 100644 --- a/packages/angular/create/BUILD.bazel +++ b/packages/angular/create/BUILD.bazel @@ -32,6 +32,7 @@ genrule( pkg_npm( name = "npm_package", + tags = ["release-package"], visibility = ["//visibility:public"], deps = [ ":README.md", diff --git a/packages/angular/pwa/BUILD.bazel b/packages/angular/pwa/BUILD.bazel index eeaf57c76d2a..58bdfea63444 100644 --- a/packages/angular/pwa/BUILD.bazel +++ b/packages/angular/pwa/BUILD.bazel @@ -88,6 +88,7 @@ pkg_npm( "//packages/angular_devkit/schematics:package.json", "//packages/schematics/angular:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":license", diff --git a/packages/angular_devkit/architect/BUILD.bazel b/packages/angular_devkit/architect/BUILD.bazel index a6e9f961bdc4..fe9eb922808d 100644 --- a/packages/angular_devkit/architect/BUILD.bazel +++ b/packages/angular_devkit/architect/BUILD.bazel @@ -112,6 +112,7 @@ pkg_npm( pkg_deps = [ "//packages/angular_devkit/core:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":architect", diff --git a/packages/angular_devkit/architect/node/BUILD.bazel b/packages/angular_devkit/architect/node/BUILD.bazel index ab98536ca739..ef463f7d018b 100644 --- a/packages/angular_devkit/architect/node/BUILD.bazel +++ b/packages/angular_devkit/architect/node/BUILD.bazel @@ -23,7 +23,6 @@ ts_library( "//packages/angular_devkit/architect", "//packages/angular_devkit/core", "//packages/angular_devkit/core/node", - "//tests/angular_devkit/architect/node/jobs:jobs_test_lib", "@npm//@types/node", "@npm//rxjs", ], @@ -40,6 +39,7 @@ ts_library( deps = [ ":node", "//packages/angular_devkit/architect", + "//tests/angular_devkit/architect/node/jobs:jobs_test_lib", ], ) diff --git a/packages/angular_devkit/architect_cli/BUILD.bazel b/packages/angular_devkit/architect_cli/BUILD.bazel index fee4938a4ae6..ff916d34d005 100644 --- a/packages/angular_devkit/architect_cli/BUILD.bazel +++ b/packages/angular_devkit/architect_cli/BUILD.bazel @@ -40,6 +40,7 @@ pkg_npm( "//packages/angular_devkit/architect:package.json", "//packages/angular_devkit/core:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":architect_cli", diff --git a/packages/angular_devkit/build_angular/BUILD.bazel b/packages/angular_devkit/build_angular/BUILD.bazel index 0565125a6ad4..5fb3680e2a6a 100644 --- a/packages/angular_devkit/build_angular/BUILD.bazel +++ b/packages/angular_devkit/build_angular/BUILD.bazel @@ -233,6 +233,7 @@ pkg_npm( "//packages/angular_devkit/core:package.json", "//packages/ngtools/webpack:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":build_angular", diff --git a/packages/angular_devkit/build_webpack/BUILD.bazel b/packages/angular_devkit/build_webpack/BUILD.bazel index 9d1cf4bb25e1..b1157c6f39a0 100644 --- a/packages/angular_devkit/build_webpack/BUILD.bazel +++ b/packages/angular_devkit/build_webpack/BUILD.bazel @@ -120,6 +120,7 @@ pkg_npm( pkg_deps = [ "//packages/angular_devkit/architect:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":build_webpack", diff --git a/packages/angular_devkit/core/BUILD.bazel b/packages/angular_devkit/core/BUILD.bazel index 8592428c20c4..f1807387e7c0 100644 --- a/packages/angular_devkit/core/BUILD.bazel +++ b/packages/angular_devkit/core/BUILD.bazel @@ -83,6 +83,7 @@ genrule( pkg_npm( name = "npm_package", + tags = ["release-package"], deps = [ ":README.md", ":core", diff --git a/packages/angular_devkit/schematics/BUILD.bazel b/packages/angular_devkit/schematics/BUILD.bazel index 9f56a7377905..bf0b3f8401b8 100644 --- a/packages/angular_devkit/schematics/BUILD.bazel +++ b/packages/angular_devkit/schematics/BUILD.bazel @@ -86,6 +86,7 @@ pkg_npm( pkg_deps = [ "//packages/angular_devkit/core:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":collection-schema.json", diff --git a/packages/angular_devkit/schematics_cli/BUILD.bazel b/packages/angular_devkit/schematics_cli/BUILD.bazel index 9e4af505cfeb..ebd2b1e54ae0 100644 --- a/packages/angular_devkit/schematics_cli/BUILD.bazel +++ b/packages/angular_devkit/schematics_cli/BUILD.bazel @@ -110,6 +110,7 @@ pkg_npm( "//packages/angular_devkit/schematics:package.json", "//packages/angular_devkit/core:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":license", diff --git a/packages/ngtools/webpack/BUILD.bazel b/packages/ngtools/webpack/BUILD.bazel index 53b8a5d12abf..49899710c2e6 100644 --- a/packages/ngtools/webpack/BUILD.bazel +++ b/packages/ngtools/webpack/BUILD.bazel @@ -85,6 +85,7 @@ genrule( pkg_npm( name = "npm_package", + tags = ["release-package"], deps = [ ":README.md", ":license", diff --git a/packages/schematics/angular/BUILD.bazel b/packages/schematics/angular/BUILD.bazel index 83da30a8bacc..39640a9e8a71 100644 --- a/packages/schematics/angular/BUILD.bazel +++ b/packages/schematics/angular/BUILD.bazel @@ -166,6 +166,7 @@ pkg_npm( "//packages/angular_devkit/schematics:package.json", "//packages/angular_devkit/core:package.json", ], + tags = ["release-package"], deps = [ ":README.md", ":angular", diff --git a/scripts/build-packages-dist.mts b/scripts/build-packages-dist.mts new file mode 100644 index 000000000000..35de6abbfd42 --- /dev/null +++ b/scripts/build-packages-dist.mts @@ -0,0 +1,168 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +/** + * Script that builds the release output of all packages which have the "release-package + * Bazel tag set. The script builds all those packages and copies the release output to the + * distribution folder within the project. + */ + +import { BuiltPackage } from '@angular/ng-dev'; +import { execSync } from 'node:child_process'; +import { chmodSync, copyFileSync, mkdirSync, rmSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import sh from 'shelljs'; + +/** Name of the Bazel tag that will be used to find release package targets. */ +const releaseTargetTag = 'release-package'; + +/** Path to the project directory. */ +const projectDir = join(dirname(fileURLToPath(import.meta.url)), '../'); + +/** Command that runs Bazel. */ +const bazelCmd = process.env.BAZEL || `yarn -s bazel`; + +/** Command that queries Bazel for all release package targets. */ +const queryPackagesCmd = + `${bazelCmd} query --output=label "attr('tags', '\\[.*${releaseTargetTag}.*\\]', //packages/...) ` + + `intersect kind('pkg_npm', //packages/...)"`; + +/** Path for the default distribution output directory. */ +const defaultDistPath = join(projectDir, 'dist/releases'); + +/** Builds the release packages for NPM. */ +export function performNpmReleaseBuild(): BuiltPackage[] { + return buildReleasePackages(defaultDistPath, /* isSnapshotBuild */ false); +} + +/** + * Builds the release packages as snapshot build. This means that the current + * Git HEAD SHA is included in the version (for easier debugging and back tracing). + */ +export function performDefaultSnapshotBuild(): BuiltPackage[] { + return buildReleasePackages(defaultDistPath, /* isSnapshotBuild */ true); +} + +/** + * Builds the release packages with the given compile mode and copies + * the package output into the given directory. + */ +function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): BuiltPackage[] { + console.log('######################################'); + console.log(' Building release packages...'); + console.log('######################################'); + + // List of targets to build. e.g. "packages/angular/cli:npm_package" + const targets = exec(queryPackagesCmd, true).split(/\r?\n/); + const packageNames = getPackageNamesOfTargets(targets); + const bazelBinPath = exec(`${bazelCmd} info bazel-bin`, true); + const getBazelOutputPath = (pkgName: string) => + join(bazelBinPath, 'packages', pkgName, 'npm_package'); + const getDistPath = (pkgName: string) => join(distPath, pkgName); + + // Build with "--config=release" or `--config=snapshot` so that Bazel + // runs the workspace stamping script. The stamping script ensures that the + // version placeholder is populated in the release output. + const stampConfigArg = `--config=${isSnapshotBuild ? 'snapshot' : 'release'}`; + + // Walk through each release package and clear previous "npm_package" outputs. This is + // a workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1219. We need to + // do this to ensure that the version placeholders are properly populated. + packageNames.forEach((pkgName) => { + // Directory output is created by the npm_package target + const directoryOutputPath = getBazelOutputPath(pkgName); + // Archive output is created by the npm_package_archive target + const archiveOutputPath = directoryOutputPath + '_archive.tgz'; + + if (sh.test('-d', directoryOutputPath)) { + sh.chmod('-R', 'u+w', directoryOutputPath); + sh.rm('-rf', directoryOutputPath); + } + try { + chmodSync(archiveOutputPath, '0755'); + rmSync(archiveOutputPath, { force: true }); + } catch {} + }); + + // Build both the npm_package and npm_package_archive targets for each package + // TODO: Consider switching to only using the archive for publishing + const buildTargets = targets.flatMap((target) => [target, target + '_archive']); + exec(`${bazelCmd} build ${stampConfigArg} ${buildTargets.join(' ')}`); + + // Delete the distribution directory so that the output is guaranteed to be clean. Re-create + // the empty directory so that we can copy the release packages into it later. + rmSync(distPath, { force: true, recursive: true, maxRetries: 3 }); + mkdirSync(distPath, { recursive: true }); + + // Copy the package output into the specified distribution folder. + packageNames.forEach((pkgName) => { + // Directory output is created by the npm_package target + const directoryOutputPath = getBazelOutputPath(pkgName); + // Archive output is created by the npm_package_archive target + const archiveOutputPath = directoryOutputPath + '_archive.tgz'; + + const targetFolder = getDistPath(pkgName); + console.log(`> Copying package output to "${targetFolder}"`); + + // Ensure package scope directory exists prior to copying + mkdirSync(dirname(targetFolder), { recursive: true }); + + // Copy package contents to target directory + sh.cp('-R', directoryOutputPath, targetFolder); + sh.chmod('-R', 'u+w', targetFolder); + + // Copy archive of package to target directory + const archiveTargetPath = join(distPath, `${pkgName.replace('/', '_')}.tgz`); + copyFileSync(archiveOutputPath, archiveTargetPath); + chmodSync(archiveTargetPath, '0755'); + }); + + return packageNames.map((pkg) => { + return { + // Package names on disk do not have the @ scope prefix and use underscores instead of dashes + name: `@${pkg.replace(/_/g, '-')}`, + outputPath: getDistPath(pkg), + }; + }); +} + +/** + * Gets the package names of the specified Bazel targets. + * e.g. //packages/angular/cli:npm_package = angular/cli + */ +function getPackageNamesOfTargets(targets: string[]): string[] { + return targets.map((targetName) => { + const matches = targetName.match(/\/\/packages\/(.*?):/); + if (matches === null) { + throw Error( + `Found Bazel target with "${releaseTargetTag}" tag, but could not ` + + `determine release output name: ${targetName}`, + ); + } + + return matches[1]; + }); +} + +/** Executes the given command in the project directory. */ +function exec(command: string): void; +/** Executes the given command in the project directory and returns its stdout. */ +function exec(command: string, captureStdout: true): string; +function exec(command: string, captureStdout?: true) { + const stdout = execSync(command, { + cwd: projectDir, + stdio: ['inherit', captureStdout ? 'pipe' : 'inherit', 'inherit'], + }); + + if (captureStdout) { + process.stdout.write(stdout); + + return stdout.toString().trim(); + } +} diff --git a/tsconfig-build.json b/tsconfig-build.json index 025d28ebb13b..4b248abd099c 100644 --- a/tsconfig-build.json +++ b/tsconfig-build.json @@ -21,6 +21,7 @@ "tests/**/*", "tools/**/*", ".ng-dev/**/*", - "**/*_spec.ts" + "**/*_spec.ts", + "scripts/**/*.mts" ] } diff --git a/yarn.lock b/yarn.lock index 163f06bf58dd..7f03e1399f8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -121,7 +121,6 @@ "@angular/build-tooling@https://github.com/angular/dev-infra-private-build-tooling-builds.git#0940d31b019e41ca5283c1758d74fd648afd6a42": version "0.0.0-f7d207b79899e466f312ae4fea2804f7978278ad" - uid "0940d31b019e41ca5283c1758d74fd648afd6a42" resolved "https://github.com/angular/dev-infra-private-build-tooling-builds.git#0940d31b019e41ca5283c1758d74fd648afd6a42" dependencies: "@angular-devkit/build-angular" "15.1.0-next.3" @@ -284,7 +283,6 @@ "@angular/ng-dev@https://github.com/angular/dev-infra-private-ng-dev-builds.git#9dccce65ac86e8bb222f600d978b713f8e19c2f8": version "0.0.0-f7d207b79899e466f312ae4fea2804f7978278ad" - uid "9dccce65ac86e8bb222f600d978b713f8e19c2f8" resolved "https://github.com/angular/dev-infra-private-ng-dev-builds.git#9dccce65ac86e8bb222f600d978b713f8e19c2f8" dependencies: "@yarnpkg/lockfile" "^1.1.0" @@ -3088,7 +3086,7 @@ "@types/qs" "*" "@types/serve-static" "*" -"@types/glob@^8.0.0": +"@types/glob@*", "@types/glob@^8.0.0": version "8.0.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-8.0.0.tgz#321607e9cbaec54f687a0792b2d1d370739455d2" integrity sha512-l6NQsDDyQUVeoTynNpC9uRvCUint/gSUXQA2euwmTuWGvPY5LSDUu6tkCtJB2SvGQlJQzLaKqcGZP4//7EDveA== @@ -3356,6 +3354,14 @@ "@types/mime" "*" "@types/node" "*" +"@types/shelljs@^0.8.11": + version "0.8.11" + resolved "https://registry.yarnpkg.com/@types/shelljs/-/shelljs-0.8.11.tgz#17a5696c825974e96828e96e89585d685646fcb8" + integrity sha512-x9yaMvEh5BEaZKeVQC4vp3l+QoFj3BXcd4aYfuKSzIIyihjdVARAadYy3SMNIz0WCCdS2vB9JL/U6GQk5PaxQw== + dependencies: + "@types/glob" "*" + "@types/node" "*" + "@types/sockjs@^0.3.33": version "0.3.33" resolved "https://registry.yarnpkg.com/@types/sockjs/-/sockjs-0.3.33.tgz#570d3a0b99ac995360e3136fd6045113b1bd236f" @@ -10164,7 +10170,6 @@ sass@1.56.2, sass@^1.55.0: "sauce-connect-proxy@https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz": version "0.0.0" - uid "9c16682e4c9716734432789884f868212f95f563" resolved "https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz#9c16682e4c9716734432789884f868212f95f563" saucelabs@^1.5.0: