ci: enable E2E tests on multiple node versions

This commit sets ups Node 18 tool chain and also configures E2E tests to use multiple tools chains to retain the same behaviour prior to switching the e2e tests to Bazel.
angular · Feb 10, 2023 · b467bdf · b467bdf
1 parent 5707ffa
commit b467bdf
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 20 deletions.
diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml
@@ -29,7 +29,7 @@ var_3: &default_nodeversion '14.20'
 var_3_major: &default_nodeversion_major '14'
 # The major version of node toolchains. See tools/toolchain_info.bzl
 # NOTE: entries in this array may be repeated elsewhere in the file, find them before adding more
-var_3_all_major: &all_nodeversion_major ['14', '16']
+var_3_all_major: &all_nodeversion_major ['14', '16', '18']
 # Workspace initially persisted by the `setup` job, and then enhanced by `setup-and-build-win`.
 # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
 # https://circleci.com/blog/deep-diving-into-circleci-workspaces/
@@ -236,7 +236,7 @@ jobs:
             equal: [*default_nodeversion_major, << parameters.nodeversion >>]
           steps:
             - run:
-                command: yarn bazel test --test_tag_filters=-node16,-node<< parameters.nodeversion >>-broken //packages/...
+                command: yarn bazel test --test_tag_filters=-node16,-node18,-node<< parameters.nodeversion >>-broken //packages/...
                 # This timeout provides time for the actual tests to timeout and report status
                 # instead of CircleCI stopping the job without test failure information.
                 no_output_timeout: 40m
@@ -257,6 +257,9 @@ jobs:
     executor: bazel-executor
     parallelism: 8
     parameters:
+      nodeversion:
+        type: string
+        default: *default_nodeversion
       snapshots:
         type: boolean
         default: false
@@ -271,12 +274,12 @@ jobs:
       - run: mkdir /mnt/ramdisk/e2e
       - run:
           name: Execute CLI E2E Tests with << parameters.subset >>
-          command: yarn bazel test --define=E2E_TEMP=/mnt/ramdisk/e2e --define=E2E_SHARD_TOTAL=${CIRCLE_NODE_TOTAL} --define=E2E_SHARD_INDEX=${CIRCLE_NODE_INDEX} --config=e2e //tests/legacy-cli:e2e<<# parameters.snapshots >>.snapshots<</ parameters.snapshots >>.<< parameters.subset >>
+          command: yarn bazel test --define=E2E_TEMP=/mnt/ramdisk/e2e --define=E2E_SHARD_TOTAL=${CIRCLE_NODE_TOTAL} --define=E2E_SHARD_INDEX=${CIRCLE_NODE_INDEX} --config=e2e //tests/legacy-cli:e2e<<# parameters.snapshots >>.snapshots<</ parameters.snapshots >>.<< parameters.subset >>_node<< parameters.nodeversion >>
           no_output_timeout: 40m
       - store_artifacts:
-          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+          path: dist/testlogs/tests/legacy-cli/e2e.<<parameters.nodeversion>>.<< parameters.subset >>
       - store_test_results:
-          path: dist/testlogs/tests/legacy-cli/e2e.<< parameters.subset >>
+          path: dist/testlogs/tests/legacy-cli/e2e.<<parameters.nodeversion>>.<< parameters.subset >>
       - fail_fast
 
   test-browsers:
@@ -367,7 +370,7 @@ jobs:
             PWD: .
           command: |
             mkdir X:/ramdisk/e2e
-            bazel test --define=E2E_TEMP=X:/ramdisk/e2e --define=E2E_SHARD_TOTAL=$env:CIRCLE_NODE_TOTAL --define=E2E_SHARD_INDEX=$env:CIRCLE_NODE_INDEX --config=e2e //tests/legacy-cli:e2e.npm
+            bazel test --define=E2E_TEMP=X:/ramdisk/e2e --define=E2E_SHARD_TOTAL=$env:CIRCLE_NODE_TOTAL --define=E2E_SHARD_INDEX=$env:CIRCLE_NODE_INDEX --config=e2e //tests/legacy-cli:e2e.npm_node16
           # This timeout provides time for the actual tests to timeout and report status
           # instead of CircleCI stopping the job without test failure information.
           no_output_timeout: 40m
@@ -396,7 +399,10 @@ workflows:
           name: test-node<< matrix.nodeversion >>
           matrix:
             parameters:
-              nodeversion: *all_nodeversion_major
+              # Disable running this on Node.js 18 for now. This causes failures when using RBE.
+              # Example: `GLIBC_2.28' not found.
+              # Likely this is caused by outdated Ubuntu image https://github.com/angular/dev-infra/blob/main/bazel/remote-execution/BUILD.bazel#L21
+              nodeversion: ['14', '16']
           requires:
             - build
 
@@ -405,18 +411,34 @@ workflows:
           requires:
             - setup
 
+      - e2e-tests:
+          name: e2e-cli-<< matrix.subset >>
+          nodeversion: *default_nodeversion_major
+          matrix:
+            parameters:
+              subset: *all_e2e_subsets
+          filters:
+            branches:
+              ignore:
+                - main
+                - /\d+\.\d+\.x/
+          requires:
+            - build
+
       - e2e-tests:
           name: e2e-cli-node-<<matrix.nodeversion>>-<< matrix.subset >>
           matrix:
             alias: e2e-cli
             parameters:
+              nodeversion: *all_nodeversion_major
               subset: *all_e2e_subsets
-          snapshots: false
           requires:
             - build
+          <<: *only_release_branches
 
       - e2e-tests:
           name: e2e-snapshots-<< matrix.subset >>
+          nodeversion: *default_nodeversion_major
           matrix:
             parameters:
               subset: *all_e2e_subsets

diff --git a/WORKSPACE b/WORKSPACE
@@ -99,6 +99,11 @@ nodejs_register_toolchains(
     node_version = "16.13.1",
 )
 
+nodejs_register_toolchains(
+    name = "node18",
+    node_version = "18.10.0",
+)
+
 register_toolchains(
     "@npm//@angular/build-tooling/bazel/git-toolchain:git_linux_toolchain",
     "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_x86_toolchain",

diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md
@@ -86,8 +86,8 @@ You can find more info about debugging [tests with Bazel in the docs.](https://g
 ### End to end tests
 
 - For a complete list of test targets use the following Bazel query: `yarn bazel query "tests(//tests/...)"`
-- Run a subset of the tests: `yarn bazel test //tests/legacy-cli:e2e --test_filter="tests/i18n/ivy-localize-*"`
-- Use `bazel run` to debug failing tests debugging: `yarn bazel run //tests/legacy-cli:e2e --test_arg="--glob=tests/basic/aot.ts"`
+- Run a subset of the tests: `yarn bazel test //tests/legacy-cli:e2e_node16 --test_filter="tests/i18n/ivy-localize-*"`
+- Use `bazel run` to debug failing tests debugging: `yarn bazel run //tests/legacy-cli:e2e_node16 --test_arg="--glob=tests/basic/aot.ts"`
 - Provide additional `e2e_runner` using `--test_arg`: `--test_arg="--yarn"`
 
 When running the debug commands, Node will stop and wait for a debugger to attach.

diff --git a/tests/legacy-cli/e2e.bzl b/tests/legacy-cli/e2e.bzl
@@ -1,4 +1,5 @@
 load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_test")
+load("//tools:toolchain_info.bzl", "TOOLCHAINS_NAMES", "TOOLCHAINS_VERSIONS")
 
 # bazel query --output=label "kind('pkg_tar', //packages/...)"
 TESTED_PACKAGES = [
@@ -54,14 +55,19 @@ def e2e_suites(name, runner, data):
         data: runtime deps such as tests and test data
     """
 
-    # Default target meant to be run manually for debugging, customizing test cli via bazel
-    _e2e_tests(name, runner = runner, data = data, tags = ["manual"])
-
     # Pre-configured test suites
-    # TODO: add node 14 + 16
-    _e2e_suite(name, runner, "npm", data)
-    _e2e_suite(name, runner, "yarn", data)
-    _e2e_suite(name, runner, "esbuild", data)
+    for toolchain_name, toolchain in zip(
+        TOOLCHAINS_NAMES,
+        TOOLCHAINS_VERSIONS,
+    ):
+        # Default target meant to be run manually for debugging, customizing test cli via bazel
+        _e2e_tests(name + "_" + toolchain_name, runner, data = data, toolchain = toolchain, tags = ["manual"])
+
+        _e2e_suite(name, runner, "npm", data, toolchain_name, toolchain)
+        _e2e_suite(name, runner, "yarn", data, toolchain_name, toolchain)
+        _e2e_suite(name, runner, "esbuild", data, toolchain_name, toolchain)
+
+    # Saucelabs tests are only run on the default toolchain
     _e2e_suite(name, runner, "saucelabs", data)
 
 def _e2e_tests(name, runner, **kwargs):
@@ -107,14 +113,17 @@ def _e2e_tests(name, runner, **kwargs):
         **kwargs
     )
 
-def _e2e_suite(name, runner, type, data):
+def _e2e_suite(name, runner, type, data, toolchain_name = "", toolchain = None):
     """
     Setup a predefined test suite (yarn|esbuild|saucelabs|npm).
     """
     args = []
     tests = None
     ignore = None
 
+    if toolchain_name:
+        toolchain_name = "_" + toolchain_name
+
     if type == "yarn":
         args.append("--yarn")
         tests = YARN_TESTS
@@ -132,10 +141,11 @@ def _e2e_suite(name, runner, type, data):
 
     # Standard e2e tests
     _e2e_tests(
-        name = "%s.%s" % (name, type),
+        name = "%s.%s%s" % (name, type, toolchain_name),
         runner = runner,
         size = "enormous",
         data = data,
+        toolchain = toolchain,
         shard_count = TEST_SHARD_COUNT,
         templated_args = [
             "--glob=%s" % _to_glob(tests) if tests else "",
@@ -145,10 +155,11 @@ def _e2e_suite(name, runner, type, data):
 
     # e2e tests of snapshot builds
     _e2e_tests(
-        name = "%s.snapshots.%s" % (name, type),
+        name = "%s.snapshots.%s%s" % (name, type, toolchain_name),
         runner = runner,
         size = "enormous",
         data = data,
+        toolchain = toolchain,
         shard_count = TEST_SHARD_COUNT,
         templated_args = [
             "--ng-snapshots",

diff --git a/tools/toolchain_info.bzl b/tools/toolchain_info.bzl
@@ -5,6 +5,7 @@
 TOOLCHAINS_NAMES = [
     "node14",
     "node16",
+    "node18",
 ]
 
 # this is the list of toolchains that should be used and are registered with nodejs_register_toolchains in the WORKSPACE file
@@ -19,6 +20,11 @@ TOOLCHAINS_VERSIONS = [
         "@bazel_tools//src/conditions:darwin": "@node16_darwin_amd64//:node_toolchain",
         "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_toolchain",
     }),
+    select({
+        "@bazel_tools//src/conditions:linux_x86_64": "@node18_linux_amd64//:node_toolchain",
+        "@bazel_tools//src/conditions:darwin": "@node18_darwin_amd64//:node_toolchain",
+        "@bazel_tools//src/conditions:windows": "@node18_windows_amd64//:node_toolchain",
+    }),
 ]
 
 # A default toolchain for use when only one is necessary