test: saucelab targets for all karma tests (angular#35516)

PR Close angular#35516

Author: gregmagolan

.circleci/config.yml

@@ -343,10 +343,6 @@ jobs:
           path: dist/bin/packages/core/test/bundling/todo/bundle.min.js.br
           destination: core/todo/bundle.br
 
-  # This job is currently a PoC for running tests on SauceLabs via bazel. It runs a subset of the
-  # tests in `legacy-unit-tests-saucelabs` (see
-  # [BUILD.bazel](https://github.com/angular/angular/blob/ef44f51d5/BUILD.bazel#L66-L92)).
-  #
   # NOTE: This is currently limited to master builds only. See the `default_workflow` configuration.
   saucelabs_view_engine:
     executor:
@@ -360,16 +356,18 @@ jobs:
       - init_environment
       - init_saucelabs_environment
       - run:
-          name: Run Bazel tests on Saucelabs
+          name: Run Bazel tests on Saucelabs with ViewEngine
           # See /tools/saucelabs/README.md for more info
           command: |
             yarn bazel run //tools/saucelabs:sauce_service_setup
-            yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
+            TESTS=$(./node_modules/.bin/bazel query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "ivy-only", ...) except attr("tags", "fixme-saucelabs", ...)')
+            yarn bazel test --config=saucelabs ${TESTS}
             yarn bazel run //tools/saucelabs:sauce_service_stop
-          no_output_timeout: 20m
+          no_output_timeout: 40m
       - notify_webhook_on_fail:
           webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
 
+  # NOTE: This is currently limited to master builds only. See the `default_workflow` configuration.
   saucelabs_ivy:
     executor:
       name: default-executor
@@ -382,13 +380,16 @@ jobs:
       - init_environment
       - init_saucelabs_environment
       - run:
-          name: Run Bazel tests on Saucelabs
+          name: Run Bazel tests on Saucelabs with Ivy
           # See /tools/saucelabs/README.md for more info
           command: |
             yarn bazel run //tools/saucelabs:sauce_service_setup
-            yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
+            TESTS=$(./node_modules/.bin/bazel query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "no-ivy-aot", ...) except attr("tags", "fixme-saucelabs", ...)')
+            yarn bazel test --config=saucelabs --config=ivy ${TESTS}
             yarn bazel run //tools/saucelabs:sauce_service_stop
-          no_output_timeout: 20m
+          no_output_timeout: 40m
+      - notify_webhook_on_fail:
+          webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
 
   test_aio:
     executor: default-executor
@@ -840,13 +841,22 @@ workflows:
           requires:
             - setup
       - saucelabs_ivy:
+          # Testing saucelabs via Bazel currently taking longer than the legacy saucelabs job as it
+          # each karma_web_test target is provisioning and tearing down browsers which is adding
+          # a lot of overhead. Running on master only to avoid wasting resources and slowing down
+          # CI for PRs.
+          # TODO: Run this job on all branches (including PRs) once karma_web_test targets can
+          # share provisioned browsers and we can remove the legacy saucelabs job.
+          <<: *only_on_master
           requires:
-            - test_ivy_aot
+            - setup
       - saucelabs_view_engine:
-          # This job is currently a PoC and a subset of `legacy-unit-tests-saucelabs`. Running on
-          # master only to avoid wasting resources.
-          # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC and
-          # we can remove the legacy saucelabs job.
+          # Testing saucelabs via Bazel currently taking longer than the legacy saucelabs job as it
+          # each karma_web_test target is provisioning and tearing down browsers which is adding
+          # a lot of overhead. Running on master only to avoid wasting resources and slowing down
+          # CI for PRs.
+          # TODO: Run this job on all branches (including PRs) once karma_web_test targets can
+          # share provisioned browsers and we can remove the legacy saucelabs job.
           <<: *only_on_master
           requires:
             - setup
@@ -899,6 +909,7 @@ workflows:
             - test
             - test_ivy_aot
             - integration_test
+            - saucelabs_view_engine
             - saucelabs_ivy
             # Only publish if `aio`/`docs` tests using the locally built Angular packages pass
             - test_aio_local

BUILD.bazel

@@ -1,7 +1,5 @@
 package(default_visibility = ["//visibility:public"])
 
-load("//tools:defaults.bzl", "karma_web_test")
-
 exports_files([
     "LICENSE",
     "protractor-perf.conf.js",
@@ -46,76 +44,3 @@ filegroup(
         "@npm//:node_modules/angular-mocks-1.6/angular-mocks.js",
     ],
 )
-
-# To run manually:
-# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
-# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
-# environment variables and run:
-# ```
-# yarn bazel run //tools/saucelabs:sauce_service_setup
-# yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
-# ```
-# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
-karma_web_test(
-    name = "saucelabs_unit_tests",
-    # Default timeout is moderate (5min). This causes the test to be terminated while
-    # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
-    # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
-    # Bazel's long timeout (15min). This ensures that Karma can shut down properly.
-    timeout = "long",
-    karma = "//tools/saucelabs:karma-saucelabs",
-    tags = [
-        "manual",
-        "no-remote-exec",
-        "saucelabs",
-    ],
-    deps = [
-        "//packages/core/test/acceptance:acceptance_lib",
-    ],
-)
-
-SAUCE_TEST_SUITE_TARGETS = [
-    "packages/common/http/test:test_lib",
-    "packages/common/http/testing/test:test_lib",
-    "packages/common/test:test_lib",
-    "packages/core/test:test_lib",
-    "packages/forms/test:test_lib",
-    "packages/http/test:test_lib",
-]
-
-[
-    # These target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
-    # subset of the legacy saucelabs tests.
-    karma_web_test(
-        name = "saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_"),
-        # Default timeout is moderate (5min). This causes the test to be terminated while
-        # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
-        # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
-        # Bazel's long timeout (15min). This ensures that Karma can shut down properly.
-        timeout = "long",
-        karma = "//tools/saucelabs:karma-saucelabs",
-        tags = [
-            "exclusive",
-            "manual",
-            "no-remote-exec",
-            "saucelabs",
-        ],
-        deps = ["//%s" % test],
-    )
-    for test in SAUCE_TEST_SUITE_TARGETS
-]
-
-# To run manually:
-# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
-# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
-# environment variables and run:
-# ```
-# yarn bazel run //tools/saucelabs:sauce_service_setup
-# yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
-# ```
-# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
-test_suite(
-    name = "saucelabs_unit_tests_poc_suite",
-    tags = ["manual"],
-    tests = ["//:saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_") for test in SAUCE_TEST_SUITE_TARGETS],
-)

tools/defaults.bzl

@@ -214,54 +214,69 @@ def pkg_npm(name, substitutions = {}, **kwargs):
         visibility = visibility,
     )
 
-def karma_web_test(bootstrap = [], deps = [], data = [], runtime_deps = [], **kwargs):
-    """Default values for karma_web_test"""
-    if not bootstrap:
-        bootstrap = ["//:web_test_bootstrap_scripts"]
-    local_deps = [
+def karma_web_test_suite(name, **kwargs):
+    """Default values for karma_web_test_suite"""
+
+    # Default value for bootstrap
+    bootstrap = kwargs.pop("bootstrap", [
+        "//:web_test_bootstrap_scripts",
+    ])
+
+    # Add common deps
+    deps = kwargs.pop("deps", []) + [
         "@npm//karma-browserstack-launcher",
         "@npm//karma-sauce-launcher",
         "@npm//:node_modules/tslib/tslib.js",
         "//tools/rxjs:rxjs_umd_modules",
         "//packages/zone.js:npm_package",
-    ] + deps
-    local_runtime_deps = [
+    ]
+
+    # Add common runtime deps
+    runtime_deps = kwargs.pop("runtime_deps", []) + [
         "//tools/testing:browser",
-    ] + runtime_deps
+    ]
+
+    data = kwargs.pop("data", [])
+    tags = kwargs.pop("tags", [])
 
+    _karma_web_test_suite(
+        name = name,
+        runtime_deps = runtime_deps,
+        bootstrap = bootstrap,
+        deps = deps,
+        browsers = ["//tools/browsers:chromium"],
+        data = data,
+        tags = tags,
+        **kwargs
+    )
+
+    # Add a saucelabs target for these karma tests
     _karma_web_test(
-        runtime_deps = local_runtime_deps,
+        name = "saucelabs_%s" % name,
+        # Default timeout is moderate (5min). This causes the test to be terminated while
+        # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
+        # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
+        # Bazel's long timeout (15min). This ensures that Karma can shut down properly.
+        timeout = "long",
+        runtime_deps = runtime_deps,
         bootstrap = bootstrap,
         config_file = "//:karma-js.conf.js",
-        deps = local_deps,
+        deps = deps,
         data = data + [
             "//:browser-providers.conf.js",
             "//tools:jasmine-seed-generator.js",
         ],
+        karma = "//tools/saucelabs:karma-saucelabs",
+        tags = tags + [
+            "exclusive",
+            "manual",
+            "no-remote-exec",
+            "saucelabs",
+        ],
         configuration_env_vars = ["KARMA_WEB_TEST_MODE"],
         **kwargs
     )
 
-def karma_web_test_suite(bootstrap = [], deps = [], runtime_deps = [], **kwargs):
-    """Default values for karma_web_test_suite"""
-    if not bootstrap:
-        bootstrap = ["//:web_test_bootstrap_scripts"]
-    local_deps = [
-        "@npm//:node_modules/tslib/tslib.js",
-        "//tools/rxjs:rxjs_umd_modules",
-    ] + deps
-    local_runtime_deps = [
-        "//tools/testing:browser",
-    ] + runtime_deps
-
-    _karma_web_test_suite(
-        runtime_deps = local_runtime_deps,
-        bootstrap = bootstrap,
-        deps = local_deps,
-        browsers = ["//tools/browsers:chromium"],
-        **kwargs
-    )
-
 def protractor_web_test_suite(**kwargs):
     """Default values for protractor_web_test_suite"""