Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ci: share data between jobs using workspaces #28928

Closed
wants to merge 1 commit into from
+77 −93
Closed

ci: share data between jobs using workspaces #28928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions .bazelignore
View file
Open in desktop
@@ -1,5 +1,6 @@
node_modules
dist
aio/content
aio/node_modules
aio/tools/examples/shared/node_modules
integration/bazel
169 changes: 76 additions & 93 deletions .circleci/config.yml
View file
Open in desktop
Expand Up @@ -26,7 +26,7 @@ var_2: &browsers_docker_image circleci/node:10.12-browsers
# **NOTE 1 **: If you change the cache key prefix, also sync the restore_cache fallback to match.
# **NOTE 2 **: Keep the static part of the cache key as prefix to enable correct fallbacks.
# See https://circleci.com/docs/2.0/caching/#restoring-cache for how prefixes work in CircleCI.
var_3: &cache_key v2-angular-node-10.12-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}
var_3: &cache_key v3-angular-node-10.12-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}-{{ checksum "aio/yarn.lock" }}

# Initializes the CI environment by setting up common environment variables.
var_4: &init_environment
Expand All @@ -40,6 +40,15 @@ var_4: &init_environment
sudo ln -fs $ourYarn /usr/local/bin/yarn
echo "Yarn version: $(yarn --version)"
# Add GitHub to known hosts.
mkdir -p ~/.ssh
echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> ~/.ssh/known_hosts
# use git+ssh instead of https
git config --global url."ssh://git@github.com".insteadOf "https://github.com" || true
git config --global gc.auto 0 || true
var_5: &setup_bazel_remote_execution
run:
name: "Setup bazel RBE remote execution"
Expand Down Expand Up @@ -97,7 +106,7 @@ var_10: &restore_cache
keys:
- *cache_key
# This fallback should be the cache_key without variables.
- v2-angular-node-10.12-
- v3-angular-node-10.12-

# Branch filter that can be specified for jobs that should only run on publish branches
# (e.g. master or the patch branch)
Expand All @@ -108,16 +117,36 @@ var_12: &publish_branches_filter
# e.g. 7.0.x, 7.1.x, etc.
- /\d+\.\d+\.x/

# Workspace initially persisted by the `install` job, and then enhanced by test_aio and
# build-npm-packages.
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
# https://circleci.com/blog/deep-diving-into-circleci-workspaces/
var_13: &attach_options
at: .

version: 2
jobs:
lint:
install:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is install the right name? how about init or setup or prepare? we do much more than just "install"

<<: *job_defaults
steps:
- checkout
- *post_checkout
# This cache is saved in the build-npm-packages so that Bazel cache is also included.
- *restore_cache
- *init_environment
- *yarn_install
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
# Persist any changes at this point to be reused by further jobs.
- persist_to_workspace:
root: .
paths:
- ./*

lint:
<<: *job_defaults
steps:
- attach_workspace: *attach_options
- *init_environment

- run: 'yarn bazel:format -mode=check ||
(echo "BUILD files not formatted. Please run ''yarn bazel:format''" ; exit 1)'
Expand All @@ -131,11 +160,8 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config

# Setup remote execution and run RBE-compatible tests.
Expand All @@ -150,11 +176,8 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config
- *setup_bazel_remote_execution

Expand Down Expand Up @@ -188,9 +211,7 @@ jobs:
# Needed because the AIO tests and the PWA score test depend on Chrome being available.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
# Build aio
- run: yarn --cwd aio build --progress=false
Expand All @@ -215,9 +236,7 @@ jobs:
# Needed because before deploying the deploy-production script runs the PWA score tests.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
# Deploy angular.io to production (if necessary)
- run: setPublicVar CI_STABLE_BRANCH "$(npm info @angular/core dist-tags.latest | sed -r 's/^\s*([0-9]+\.[0-9]+)\.[0-9]+.*$/\1.x/')"
Expand All @@ -229,11 +248,7 @@ jobs:
# Needed because the AIO tests and the PWA score test depend on Chrome being available.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Build aio (with local Angular packages)
- run: yarn --cwd aio build-local --progress=false
Expand All @@ -248,23 +263,15 @@ jobs:
test_aio_local_ivy:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Build aio with Ivy (using local Angular packages)
- run: yarn --cwd aio build-with-ivy --progress=false

test_aio_tools:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Install
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
Expand All @@ -280,11 +287,7 @@ jobs:
- image: *browsers_docker_image
parallelism: 3
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Install aio
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
Expand All @@ -300,11 +303,7 @@ jobs:
- image: *browsers_docker_image
parallelism: 3
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Install aio
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
Expand All @@ -319,9 +318,7 @@ jobs:
environment:
AIO_SNAPSHOT_ARTIFACT_PATH: &aio_preview_artifact_path 'aio/tmp/snapshot.tgz'
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- run: ./aio/scripts/build-artifacts.sh $AIO_SNAPSHOT_ARTIFACT_PATH $CI_PULL_REQUEST $CI_COMMIT
- store_artifacts:
Expand All @@ -338,9 +335,7 @@ jobs:
# Needed because the test-preview script runs e2e tests and the PWA score test with Chrome.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
- run:
Expand All @@ -360,27 +355,24 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config
- *setup_bazel_remote_execution

- run: scripts/build-packages-dist.sh

# Save the npm packages from //packages/... for other workflow jobs to read
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
- persist_to_workspace:
root: dist
root: .
paths:
- packages-dist
- dist/packages-dist

- save_cache:
key: *cache_key
paths:
- "node_modules"
- "aio/node_modules"
- "~/bazel_repository_cache"


Expand All @@ -389,22 +381,18 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config
- *setup_bazel_remote_execution

- run: scripts/build-ivy-npm-packages.sh

# Save the npm packages from //packages/... for other workflow jobs to read
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
- persist_to_workspace:
root: dist
root: .
paths:
- packages-dist-ivy-aot
- dist/packages-dist-ivy-aot

# We run the integration tests outside of Bazel for now.
# They are a separate workflow job so that they can be easily re-run.
Expand All @@ -423,14 +411,8 @@ jobs:
# on a 4G worker so we use a larger machine here too.
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Some integration tests get their dependencies from the root `node_modules/`.
- *yarn_install
# Runs the integration tests in parallel across multiple CircleCI container instances. The
# amount of container nodes for this job is controlled by the "parallelism" option.
- run: ./integration/run_tests.sh ${CIRCLE_NODE_INDEX} ${CIRCLE_NODE_TOTAL}
Expand All @@ -440,8 +422,7 @@ jobs:
publish_snapshot:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- attach_workspace: *attach_options
- *init_environment
# See below - ideally this job should not trigger for non-upstream builds.
# But since it does, we have to check this condition.
Expand All @@ -453,8 +434,6 @@ jobs:
|| "$CI_REPO_OWNER" != "angular"
|| "$CI_REPO_NAME" != "angular"
]] && circleci step halt || true'
- attach_workspace:
at: dist
# CircleCI has a config setting to force SSH for all github connections
# This is not compatible with our mechanism of using a Personal Access Token
# Clear the global setting
Expand Down Expand Up @@ -492,11 +471,8 @@ jobs:
# and therefore the tunnel and Karma need to process a lot of file requests and tests.
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- run:
name: Preparing environment for running tests on Saucelabs.
command: |
Expand All @@ -517,13 +493,8 @@ jobs:
legacy-misc-tests:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- attach_workspace:
at: dist
- run: yarn gulp check-cycle
# TODO: disabled because the Bazel packages-dist does not seem to have map files for
# the ESM5/ES2015 output. See: https://github.com/angular/angular/issues/27966
Expand All @@ -537,23 +508,33 @@ jobs:
docker:
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- attach_workspace: *attach_options
- *init_environment
- attach_workspace:
at: dist
- run: ./scripts/ci/run_angular_material_unit_tests.sh

workflows:
version: 2
default_workflow:
jobs:
- lint
- test
- test_ivy_aot
- build-npm-packages
- build-ivy-npm-packages
- test_aio
- install
- lint:
requires:
- install
- test:
requires:
- install
- test_ivy_aot:
requires:
- install
- build-npm-packages:
requires:
- install
- build-ivy-npm-packages:
requires:
- install
- test_aio:
requires:
- install
- deploy_aio:
requires:
- test_aio
Expand All @@ -576,6 +557,8 @@ workflows:
requires:
- build-npm-packages
- aio_preview:
requires:
- install
# Only run on PR builds. (There can be no previews for non-PR builds.)
filters:
branches:
Expand Down