ci: publish tarballs for all Angular packages as build artifacts on PR builds

.circleci/config.yml

@@ -38,21 +38,26 @@ var_6: &material_unit_tests_cache_key_fallback v4-angular-material-
 # https://circleci.com/blog/deep-diving-into-circleci-workspaces/
 var_7: &workspace_location ~/
 
-# Branch filter that can be specified for jobs that should only run on publish branches
-# (e.g. master or the patch branch)
-var_8: &publish_branches_filter
-  branches:
-    only:
-      - master
-      # e.g. 7.0.x, 7.1.x, etc.
-      - /\d+\.\d+\.x/
-
-# Filter to ignore pull requests on a given job.
-var_9: &ignore_pull_requests
+# Filter to run a job on builds for pull requests only.
+var_8: &only_on_pull_requests
+  filters:
+    branches:
+      only:
+        - /pull\/\d+/
+
+# Filter to skip a job on builds for pull requests.
+var_9: &skip_on_pull_requests
   filters:
     branches:
       ignore:
-        - /pull\/.*/
+        - /pull\/\d+/
+
+# Filter to run a job on builds for the master branch only.
+var_10: &only_on_master
+  filters:
+    branches:
+      only:
+        - master
 
 # Executor Definitions
 # https://circleci.com/docs/2.0/reusing-config/#authoring-reusable-executors
@@ -549,6 +554,26 @@ jobs:
       # amount of container nodes for this job is controlled by the "parallelism" option.
       - run: ./integration/run_tests.sh ${CIRCLE_NODE_INDEX} ${CIRCLE_NODE_TOTAL}
 
+  # This job creates compressed tarballs (`.tgz` files) for all Angular packages and stores them as
+  # build artifacts. This makes it easy to try out changes from a PR build for testing purposes.
+  # More info CircleCI build artifacts: https://circleci.com/docs/2.0/artifacts
+  #
+  # NOTE: Currently, this job only runs for PR builds. See `publish_snapshot` for non-PR builds.
+  publish_packages_as_artifacts:
+    executor: default-executor
+    environment:
+      NG_PACKAGES_DIR: &ng_packages_dir 'dist/packages-dist'
+      NG_PACKAGES_ARCHIVES_DIR: &ng_packages_archives_dir 'dist/packages-dist-archives'
+    steps:
+      - custom_attach_workspace
+      - init_environment
+      - run:
+          name: Create artifacts
+          command: ./scripts/ci/create-package-archives.sh $CI_PULL_REQUEST $CI_COMMIT $NG_PACKAGES_DIR $NG_PACKAGES_ARCHIVES_DIR
+      - store_artifacts:
+          path: *ng_packages_archives_dir
+          destination: angular
+
   # This job updates the content of repos like github.com/angular/core-builds
   # for every green build on angular/angular.
   publish_snapshot:
@@ -770,15 +795,13 @@ workflows:
           requires:
             - setup
       - test_saucelabs_bazel:
-          requires:
-            - setup
           # This job is currently a PoC and a subset of `legacy-unit-tests-saucelabs`. Running on
           # master only to avoid wasting resources.
           #
           # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC.
-          filters:
-            branches:
-              only: master
+          <<: *only_on_master
+          requires:
+            - setup
       - test_aio:
           requires:
             - setup
@@ -805,18 +828,21 @@ workflows:
           requires:
             - build-ivy-npm-packages
       - aio_preview:
+          # Only run on PR builds. (There can be no previews for non-PR builds.)
+          <<: *only_on_pull_requests
           requires:
             - setup
-          # Only run on PR builds. (There can be no previews for non-PR builds.)
-          filters:
-            branches:
-              only: /pull\/\d+/
       - test_aio_preview:
           requires:
             - aio_preview
       - integration_test:
           requires:
             - build-npm-packages
+      - publish_packages_as_artifacts:
+          # Only run on PR builds. (For non-PR builds, the `publish_snapshot` job.)
+          <<: *only_on_pull_requests
+          requires:
+            - build-npm-packages
       - publish_snapshot:
           # Note: no filters on this job because we want it to run for all upstream branches
           # We'd really like to filter out pull requests here, but not yet available:
@@ -852,11 +878,11 @@ workflows:
       # This isn't strictly necessary as there is no artifact dependency, but helps economize
       # CI resources by not attempting to build when we know should fail.
       - test_win:
-          <<: *ignore_pull_requests
+          <<: *skip_on_pull_requests
           requires:
             - test
       - test_ivy_aot_win:
-          <<: *ignore_pull_requests
+          <<: *skip_on_pull_requests
           requires:
             - test_ivy_aot
 
@@ -871,9 +897,6 @@ workflows:
             - setup
     triggers:
       - schedule:
+          <<: *only_on_master
           # Runs AIO monitoring jobs at 10:00AM every day.
           cron: "0 10 * * *"
-          filters:
-            branches:
-              only:
-                - master

docs/DEVELOPER.md

@@ -8,6 +8,10 @@ It also explains the basic mechanics of using `git`, `node`, and `yarn`.
 * [Installing NPM Modules](#installing-npm-modules)
 * [Building](#building)
 * [Running Tests Locally](#running-tests-locally)
+* [Formatting your Source Code](#formatting-your-source-code)
+* [Linting/verifying your Source Code](#lintingverifying-your-source-code)
+* [Publishing Snapshot Builds](#publishing-snapshot-builds)
+* [Bazel Support](#bazel-support)
 
 See the [contribution guidelines](https://github.com/angular/angular/blob/master/CONTRIBUTING.md)
 if you'd like to contribute to Angular.
@@ -50,6 +54,7 @@ cd angular
 # Add the main Angular repository as an upstream remote to your repository:
 git remote add upstream https://github.com/angular/angular.git
 ```
+
 ## Installing NPM Modules
 
 Next, install the JavaScript modules needed to build and test Angular:
@@ -73,18 +78,19 @@ To build Angular run:
 
 Bazel is used as the primary tool for building and testing Angular. Building and testing is
 incremental with Bazel, and it's possible to only run tests for an individual package instead
-of for all packages. Read more about this in the [BAZEL.md](./BAZEL.md) document. 
+of for all packages. Read more about this in the [BAZEL.md](./BAZEL.md) document.
 
 You should execute all test suites before submitting a PR to GitHub:
 - `yarn bazel test packages/...`
 
 **Note**: The first test run will be much slower than future runs. This is because future runs will
-benefit from Bazel's capability to do incremental builds. 
+benefit from Bazel's capability to do incremental builds.
 
 All the tests are executed on our Continuous Integration infrastructure. PRs can only be
 merged if the code is formatted properly and all tests are passing.
 
-## <a name="clang-format"></a> Formatting your source code
+<a name="clang-format"></a>
+## Formatting your source code
 
 Angular uses [clang-format](http://clang.llvm.org/docs/ClangFormat.html) to format the source code.
 If the source code is not properly formatted, the CI will fail and the PR cannot be merged.
@@ -118,24 +124,73 @@ let g:clang_format#command = '$ANGULAR_PATH/node_modules/.bin/clang-format'
 
 where `$ANGULAR_PATH` is an environment variable of the absolute path of your Angular directory.
 
-## Linting/verifying your source code
+## Linting/verifying your Source Code
 
 You can check that your code is properly formatted and adheres to coding style by running:
 
 ``` shell
 $ yarn gulp lint
 ```
 
-## Publishing snapshot builds
+## Publishing Snapshot Builds
 
-When a build of any branch on the upstream fork angular/angular is green on CircleCI,
-it automatically publishes build artifacts
-to repositories in the Angular org, eg. the `@angular/core` package is published to
-http://github.com/angular/core-builds.
+When a build of any branch on the upstream fork angular/angular is green on CircleCI, it
+automatically publishes build artifacts to repositories in the Angular org, eg. the `@angular/core`
+package is published to https://github.com/angular/core-builds.
 
 You may find that your un-merged change needs some validation from external participants.
-Rather than requiring them to pull your Pull Request and build Angular locally, you can
-publish the `*-builds` snapshots just like our CircleCI build does.
+Rather than requiring them to pull your Pull Request and build Angular locally, they can depend on
+snapshots of the Angular packages created based on the code in the Pull Request.
+
+### Getting Packages from Build Artifacts
+Each CI run for a Pull Request stores the built Angular packages as
+[build artifacts](https://circleci.com/docs/2.0/artifacts). The artifacts are not guaranteed to be
+available as a long-term distribution mechanism, but they are guaranteed to be available around the
+time of the build.
+
+You can access the artifacts for a specific CI run by going to the workflow page, clicking on the
+`publish_packages_as_artifacts` job and then switching to the "Artifacts" tab.
+(If you happen to know the build number of the job, the URL will be something like:
+https://circleci.com/gh/angular/angular/<build-number>#artifacts)
+
+#### Archives for each Package
+On the "Artifacts" tab, there is a list of links to compressed archives for Angular packages. The
+archive names are of the format `<package-name>-pr<pr-number>-<sha>.tgz` (for example
+`core-pr12345-a1b2c3d.tgz`).
+
+One can use the URL to the `.tgz` file for each package to install them as dependencies in a
+project they need to test the Pull Request changes against. Both
+[npm](https://docs.npmjs.com/cli/install.html) and [yarn](https://yarnpkg.com/lang/en/docs/cli/add)
+support installing dependencies from URLs to `.tgz` files, for example by updating the dependencies
+in `package.json` to point to the artifact URLs and then running `npm/yarn install`:
+
+```json
+"dependencies": {
+  "@angular/common": "https://<...>.circle-artifacts.com/0/angular/common-pr12345-a1b2c3d.tgz",
+  "@angular/core": "https://<...>.circle-artifacts.com/0/angular/core-pr12345-a1b2c3d.tgz",
+  "...": "..."
+}
+```
+
+#### Download all Packages
+In addition to the individual package archives, a `.tgz` file including all packages is also
+available (named `all-pr<pr-number>-<sha>.tgz`). This can be used if one prefers to download all
+packages locally and test them by either of the following ways:
+
+1. Update the dependencies in `package.json` to point to the local uncompressed package directories.
+
+2. Directly copy the local uncompressed package directories into the `node_modules/` directory of a
+   project.
+
+Note that (while faster) the second approach has limitations. For example:
+a. Any transitive dependencies of the copied packages will not be automatically updated.
+b. The packages need to be copied over every time `npm/yarn install` is run.
+c. Some package managers (such as `pnpm` or `yarn pnp`) might not work correctly.
+
+### Publishing to GitHub repos
+You can also manually publish `*-builds` snapshots just like our CircleCI build does for upstream
+builds. Before being able to publish the packages, you need to build them locally by running the
+`./scripts/build-packages-dist.sh` script.
 
 First time, you need to create the GitHub repositories:
 
@@ -144,7 +199,7 @@ $ export TOKEN=[get one from https://github.com/settings/tokens]
 $ CREATE_REPOS=1 ./scripts/ci/publish-build-artifacts.sh [GitHub username]
 ```
 
-For subsequent snapshots, just run
+For subsequent snapshots, just run:
 
 ``` shell
 $ ./scripts/ci/publish-build-artifacts.sh [GitHub username]
@@ -153,8 +208,10 @@ $ ./scripts/ci/publish-build-artifacts.sh [GitHub username]
 The script will publish the build snapshot to a branch with the same name as your current branch,
 and create it if it doesn't exist.
 
-## Bazel support
+## Bazel Support
+
 ### IDEs
+
 #### VS Code
 
 1. Install [Bazel](https://marketplace.visualstudio.com/items?itemName=DevonDCarew.bazel-code) extension for VS Code.
@@ -173,10 +230,10 @@ build action rather than performing the build locally.
 
 > Remote Build Execution and uploading to the Remote Cache requires authentication as a google.com or angular.io account.
 
-### --config=remote-http-caching flag
+#### --config=remote-http-caching flag
 The `--config=remote-http-caching` flag can be added to enable uploading of build results to the shared http cache.  This flag
 can be added to the `.bazelrc.user` file using the script at `scripts/local-dev/setup-rbe.sh`.
 
-### --config=remote flag
-The `--config=remote` flag can be added to enable remote execution of builds.  This flag can be added to 
-the `.bazelrc.user` file using the script at `scripts/local-dev/setup-rbe.sh`.
+#### --config=remote flag
+The `--config=remote` flag can be added to enable remote execution of builds.  This flag can be added to
+the `.bazelrc.user` file using the script at `scripts/local-dev/setup-rbe.sh`.

scripts/ci/create-package-archives.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+readonly prNumber="$1"
+readonly prLastSha="${2:0:7}"
+readonly inputDir="$PROJECT_ROOT/$3"
+readonly outputDir="$PROJECT_ROOT/$4"
+readonly fileSuffix="-pr$prNumber-$prLastSha.tgz"
+
+echo "Creating compressed archives for packages in '$inputDir'."
+
+# Create or clean-up the output directory.
+echo "  Preparing output directory: $outputDir"
+rm -rf "$outputDir"
+mkdir -p "$outputDir"
+
+# Create a compressed archive containing all packages.
+# (This is useful for copying all packages into `node_modules/` (without changing `package.json`).)
+outputFileName=all$fileSuffix
+echo "  Creating archive with all packages --> '$outputFileName'..."
+tar --create --gzip --directory "$inputDir" --file "$outputDir/$outputFileName" --transform s/^\./packages/ .
+
+# Create a compressed archive for each package.
+# (This is useful for referencing the path/URL to the resulting archive in `package.json`.)
+for dir in $inputDir/*
+do
+  packageName=`basename "$dir"`
+  outputFileName="$packageName$fileSuffix"
+  outputFilePath="$outputDir/$outputFileName"
+
+  echo "  Processing package '$packageName' --> '$outputFileName'..."
+
+  tar --create --gzip --directory "$dir" --file "$outputFilePath" --transform s/^\./package/ .
+done
+
+echo "Done creating compressed archives."