Skip to content

Commit

Permalink
ci: share data between jobs using workspaces
Browse files Browse the repository at this point in the history
  • Loading branch information
@filipesilva
filipesilva committed Feb 22, 2019
1 parent c7fe3a9 commit b062782
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 91 deletions.
1 change: 1 addition & 0 deletions .bazelignore
View file
@@ -1,5 +1,6 @@
node_modules
dist
aio/content
aio/node_modules
aio/tools/examples/shared/node_modules
integration/bazel
157 changes: 66 additions & 91 deletions .circleci/config.yml
View file
Expand Up @@ -26,7 +26,7 @@ var_2: &browsers_docker_image circleci/node:10.12-browsers
# **NOTE 1 **: If you change the cache key prefix, also sync the restore_cache fallback to match.
# **NOTE 2 **: Keep the static part of the cache key as prefix to enable correct fallbacks.
# See https://circleci.com/docs/2.0/caching/#restoring-cache for how prefixes work in CircleCI.
var_3: &cache_key v2-angular-node-10.12-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}
var_3: &cache_key v3-angular-node-10.12-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "packages/bazel/package.bzl" }}-{{ checksum "aio/yarn.lock" }}

# Initializes the CI environment by setting up common environment variables.
var_4: &init_environment
Expand Down Expand Up @@ -97,7 +97,7 @@ var_10: &restore_cache
keys:
- *cache_key
# This fallback should be the cache_key without variables.
- v2-angular-node-10.12-
- v3-angular-node-10.12-

# Branch filter that can be specified for jobs that should only run on publish branches
# (e.g. master or the patch branch)
Expand All @@ -108,16 +108,36 @@ var_12: &publish_branches_filter
# e.g. 7.0.x, 7.1.x, etc.
- /\d+\.\d+\.x/

# Workspace initially persisted by the `install` job, and then enhanced by test_aio and
# build-npm-packages.
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
# https://circleci.com/blog/deep-diving-into-circleci-workspaces/
var_13: &attach_options
at: .

version: 2
jobs:
lint:
install:
<<: *job_defaults
steps:
- checkout
- *post_checkout
# This cache is saved in the build-npm-packages so that Bazel cache is also included.
- *restore_cache
- *init_environment
- *yarn_install
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
# Persist any changes at this point to be reused by further jobs.
- persist_to_workspace:
root: .
paths:
- ./*

lint:
<<: *job_defaults
steps:
- attach_workspace: *attach_options
- *init_environment

- run: 'yarn bazel:format -mode=check ||
(echo "BUILD files not formatted. Please run ''yarn bazel:format''" ; exit 1)'
Expand All @@ -131,11 +151,8 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config

# Setup remote execution and run RBE-compatible tests.
Expand All @@ -150,11 +167,8 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config
- *setup_bazel_remote_execution

Expand Down Expand Up @@ -188,9 +202,7 @@ jobs:
# Needed because the AIO tests and the PWA score test depend on Chrome being available.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
# Build aio
- run: yarn --cwd aio build --progress=false
Expand All @@ -215,9 +227,7 @@ jobs:
# Needed because before deploying the deploy-production script runs the PWA score tests.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
# Deploy angular.io to production (if necessary)
- run: setPublicVar CI_STABLE_BRANCH "$(npm info @angular/core dist-tags.latest | sed -r 's/^\s*([0-9]+\.[0-9]+)\.[0-9]+.*$/\1.x/')"
Expand All @@ -229,11 +239,7 @@ jobs:
# Needed because the AIO tests and the PWA score test depend on Chrome being available.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Build aio (with local Angular packages)
- run: yarn --cwd aio build-local --progress=false
Expand All @@ -248,23 +254,15 @@ jobs:
test_aio_local_ivy:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Build aio with Ivy (using local Angular packages)
- run: yarn --cwd aio build-with-ivy --progress=false

test_aio_tools:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Install
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
Expand All @@ -280,11 +278,7 @@ jobs:
- image: *browsers_docker_image
parallelism: 3
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Install aio
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
Expand All @@ -300,11 +294,7 @@ jobs:
- image: *browsers_docker_image
parallelism: 3
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Install aio
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
Expand All @@ -319,9 +309,7 @@ jobs:
environment:
AIO_SNAPSHOT_ARTIFACT_PATH: &aio_preview_artifact_path 'aio/tmp/snapshot.tgz'
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- run: ./aio/scripts/build-artifacts.sh $AIO_SNAPSHOT_ARTIFACT_PATH $CI_PULL_REQUEST $CI_COMMIT
- store_artifacts:
Expand All @@ -338,9 +326,7 @@ jobs:
# Needed because the test-preview script runs e2e tests and the PWA score test with Chrome.
- image: *browsers_docker_image
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- run: yarn --cwd aio install --frozen-lockfile --non-interactive
- run:
Expand All @@ -360,27 +346,24 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config
- *setup_bazel_remote_execution

- run: scripts/build-packages-dist.sh

# Save the npm packages from //packages/... for other workflow jobs to read
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
- persist_to_workspace:
root: dist
root: .
paths:
- packages-dist
- dist/packages-dist

- save_cache:
key: *cache_key
paths:
- "node_modules"
- "aio/node_modules"
- "~/bazel_repository_cache"


Expand All @@ -389,22 +372,18 @@ jobs:
<<: *job_defaults
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- *setup_circleci_bazel_config
- *setup_bazel_remote_execution

- run: scripts/build-ivy-npm-packages.sh

# Save the npm packages from //packages/... for other workflow jobs to read
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
- persist_to_workspace:
root: dist
root: .
paths:
- packages-dist-ivy-aot
- dist/packages-dist-ivy-aot

# We run the integration tests outside of Bazel for now.
# They are a separate workflow job so that they can be easily re-run.
Expand All @@ -423,14 +402,8 @@ jobs:
# on a 4G worker so we use a larger machine here too.
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace:
at: dist
- attach_workspace: *attach_options
- *init_environment
# Some integration tests get their dependencies from the root `node_modules/`.
- *yarn_install
# Runs the integration tests in parallel across multiple CircleCI container instances. The
# amount of container nodes for this job is controlled by the "parallelism" option.
- run: ./integration/run_tests.sh ${CIRCLE_NODE_INDEX} ${CIRCLE_NODE_TOTAL}
Expand All @@ -440,8 +413,7 @@ jobs:
publish_snapshot:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- attach_workspace: *attach_options
- *init_environment
# See below - ideally this job should not trigger for non-upstream builds.
# But since it does, we have to check this condition.
Expand All @@ -453,8 +425,6 @@ jobs:
|| "$CI_REPO_OWNER" != "angular"
|| "$CI_REPO_NAME" != "angular"
]] && circleci step halt || true'
- attach_workspace:
at: dist
# CircleCI has a config setting to force SSH for all github connections
# This is not compatible with our mechanism of using a Personal Access Token
# Clear the global setting
Expand Down Expand Up @@ -492,11 +462,8 @@ jobs:
# and therefore the tunnel and Karma need to process a lot of file requests and tests.
resource_class: xlarge
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- run:
name: Preparing environment for running tests on Saucelabs.
command: |
Expand All @@ -517,13 +484,8 @@ jobs:
legacy-misc-tests:
<<: *job_defaults
steps:
- checkout
- *post_checkout
- *restore_cache
- attach_workspace: *attach_options
- *init_environment
- *yarn_install
- attach_workspace:
at: dist
- run: yarn gulp check-cycle
# TODO: disabled because the Bazel packages-dist does not seem to have map files for
# the ESM5/ES2015 output. See: https://github.com/angular/angular/issues/27966
Expand All @@ -540,20 +502,31 @@ jobs:
- checkout
- *post_checkout
- *init_environment
- attach_workspace:
at: dist
- run: ./scripts/ci/run_angular_material_unit_tests.sh

workflows:
version: 2
default_workflow:
jobs:
- lint
- test
- test_ivy_aot
- build-npm-packages
- build-ivy-npm-packages
- test_aio
- install
- lint:
requires:
- install
- test:
requires:
- install
- test_ivy_aot:
requires:
- install
- build-npm-packages:
requires:
- install
- build-ivy-npm-packages:
requires:
- install
- test_aio:
requires:
- install
- deploy_aio:
requires:
- test_aio
Expand All @@ -576,6 +549,8 @@ workflows:
requires:
- build-npm-packages
- aio_preview:
requires:
- install
# Only run on PR builds. (There can be no previews for non-PR builds.)
filters:
branches:
Expand Down

0 comments on commit b062782

Please sign in to comment.