🏗 Follow up changes to integration tests on sauce labs

build-system/pr-check/remote-tests.js

@@ -29,16 +29,14 @@ const {
   stopTimer,
   startSauceConnect,
   stopSauceConnect,
-  timedExec,
   timedExecOrDie: timedExecOrDieBase,
 } = require('./utils');
 const {determineBuildTargets} = require('./build-targets');
 const {isTravisPullRequestBuild} = require('../travis');
 
 const FILENAME = 'remote-tests.js';
 const FILELOGPREFIX = colors.bold(colors.yellow(`${FILENAME}:`));
-const timedExecOrDie = (cmd, unusedFileName) =>
-  timedExecOrDieBase(cmd, FILENAME);
+const timedExecOrDie = cmd => timedExecOrDieBase(cmd, FILENAME);
 
 async function main() {
   const startTime = startTimer(FILENAME, FILENAME);
@@ -49,7 +47,10 @@ async function main() {
 
     await startSauceConnect(FILENAME);
     timedExecOrDie('gulp unit --nobuild --saucelabs');
-    timedExecOrDie('gulp integration --nobuild --compiled --saucelabs');
+    timedExecOrDie(
+      'gulp integration --nobuild --compiled --saucelabs --stable'
+    );
+    timedExecOrDie('gulp integration --nobuild --compiled --saucelabs --beta');
 
     stopSauceConnect(FILENAME);
   } else {
@@ -86,7 +87,9 @@ async function main() {
       timedExecOrDie(
         'gulp integration --nobuild --compiled --saucelabs --stable'
       );
-      timedExec('gulp integration --nobuild --compiled --saucelabs --beta');
+      timedExecOrDie(
+        'gulp integration --nobuild --compiled --saucelabs --beta'
+      );
     }
     stopSauceConnect(FILENAME);
   }

build-system/tasks/report-test-status.js

@@ -30,10 +30,15 @@ const IS_GULP_E2E = argv._[0] === 'e2e';
 
 const IS_LOCAL_CHANGES = !!argv.local_changes;
 const IS_SAUCELABS = !!argv.saucelabs;
+const IS_SAUCELABS_STABLE = !!argv.saucelabs && !!argv.stable;
+const IS_SAUCELABS_BETA = !!argv.saucelabs && !!argv.beta;
 const IS_SINGLE_PASS = !!argv.single_pass;
 
 const TEST_TYPE_SUBTYPES = new Map([
-  ['integration', ['local', 'single-pass', 'saucelabs']],
+  [
+    'integration',
+    ['local', 'single-pass', 'saucelabs-beta', 'saucelabs-stable'],
+  ],
   ['unit', ['local', 'local-changes', 'saucelabs']],
   ['e2e', ['local']],
 ]);
@@ -61,7 +66,11 @@ function inferTestType() {
     return `${type}/local-changes`;
   }
 
-  if (IS_SAUCELABS) {
+  if (IS_SAUCELABS_BETA) {
+    return `${type}/saucelabs-beta`;
+  } else if (IS_SAUCELABS_STABLE) {
+    return `${type}/saucelabs-stable`;
+  } else if (IS_SAUCELABS) {
     return `${type}/saucelabs`;
   }
 

build-system/tasks/runtime-test/helpers.js

@@ -20,7 +20,6 @@ const fs = require('fs');
 const log = require('fancy-log');
 const opn = require('opn');
 const path = require('path');
-
 const {
   reportTestErrored,
   reportTestFinished,
@@ -195,18 +194,20 @@ function karmaBrowserStart_(browser) {
  * @param {Object} browser
  * @private
  */
-function karmaBrowserComplete_(browser) {
+async function karmaBrowserComplete_(browser) {
   const result = browser.lastResult;
   result.total = result.success + result.failed + result.skipped;
-  // Prevent cases where Karma detects zero tests and still passes. #16851.
+  // Set test status to "error" if browser_complete shows zero tests (#16851).
+  // Sometimes, Sauce labs can follow this up with another successful status, in
+  // which case the error status will be replaced by a pass / fail status.
   if (result.total == 0) {
-    log(red('ERROR: Zero tests detected by Karma.'));
-    log(red(JSON.stringify(result)));
-    reportTestErrored().finally(() => {
-      if (!argv.watch) {
-        process.exit(1);
-      }
-    });
+    log(
+      yellow('WARNING:'),
+      'Received a status with zero tests:',
+      cyan(JSON.stringify(result))
+    );
+    await reportTestErrored();
+    return;
   }
   // Print a summary for each browser as soon as tests complete.
   let message =
@@ -265,7 +266,7 @@ async function runTestInSauceLabs(config) {
 
   if (argv.beta) {
     config.browsers = browsers.beta;
-    const betaExitCode = await createKarmaServer(config, () => {});
+    const betaExitCode = await createKarmaServer(config, reportTestRunComplete);
     if (betaExitCode != 0) {
       log(
         yellow('Some tests have failed on'),
@@ -300,14 +301,22 @@ async function runTestInSauceLabs(config) {
  */
 async function runTestInBatches_(config, browsers) {
   let errored = false;
-  let totalStableSuccess = 0;
-  let totalStableFailed = 0;
+  let totalSuccess = 0;
+  let totalFailed = 0;
   const partialTestRunCompleteFn = async (browsers, results) => {
     if (results.error) {
       errored = true;
     } else {
-      totalStableSuccess += results.success;
-      totalStableFailed += results.failed;
+      totalSuccess += results.success;
+      totalFailed += results.failed;
+    }
+  };
+
+  const reportResults = async () => {
+    if (errored) {
+      await reportTestErrored();
+    } else {
+      await reportTestFinished(totalSuccess, totalFailed);
     }
   };
 
@@ -318,12 +327,8 @@ async function runTestInBatches_(config, browsers) {
       config,
       partialTestRunCompleteFn
     );
-    if (errored) {
-      await reportTestErrored();
-    } else {
-      await reportTestFinished(totalStableSuccess, totalStableFailed);
-    }
     if (allBatchesExitCodes || errored) {
+      await reportResults();
       log(
         yellow('Some tests have failed on'),
         cyan('stable'),
@@ -340,7 +345,7 @@ async function runTestInBatches_(config, browsers) {
       'beta',
       browsers.beta,
       config,
-      /* runCompleteFn */ () => {}
+      partialTestRunCompleteFn
     );
     if (allBatchesExitCodes) {
       log(
@@ -356,6 +361,7 @@ async function runTestInBatches_(config, browsers) {
     }
   }
 
+  await reportResults();
   return 0;
 }