ci: attempt to increase stability of saucelabs legacy job (angular#44281)

This is an attempt to increase the stability of the Saucelabs legacy
job by using an emulator recommended by the Saucelabs platform
configurator, explicitly specifying the appium server version etc.

PR Close angular#44281

Author: devversion

Diff for: browser-providers.conf.js

@@ -10,98 +10,36 @@
 // If the target is set to null, then the browser is not run anywhere during CI.
 // If a category becomes empty (e.g. BS and required), then the corresponding job must be commented
 // out in the CI configuration.
-var CIconfiguration = {
-  // Chrome and Firefox run as part of the Bazel browser tests, so we do not run them as
-  // part of the legacy Saucelabs tests.
-  'Chrome': {unitTest: {target: null, required: false}, e2e: {target: null, required: true}},
-  'Firefox': {unitTest: {target: null, required: false}, e2e: {target: null, required: true}},
-  // Set ESR as a not required browser as it fails for Ivy acceptance tests.
-  'FirefoxESR': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: true}},
-  // Disabled because using the "beta" channel of Chrome can cause non-deterministic CI results.
-  // e.g. a new chrome beta version has been released, but the Saucelabs selenium server does
-  // not provide a chromedriver version that is compatible with the new beta.
-  'ChromeBeta': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: false}},
-  'ChromeDev': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: true}},
-  // FirefoxBeta and FirefoxDev should be target:'BS' or target:'SL', and required:true
-  // Currently deactivated due to https://github.com/angular/angular/issues/7560
-  'FirefoxBeta': {unitTest: {target: null, required: true}, e2e: {target: null, required: false}},
-  'FirefoxDev': {unitTest: {target: null, required: true}, e2e: {target: null, required: true}},
-  'Edge': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: true}},
-  'Android10': {unitTest: {target: 'SL', required: true}, e2e: {target: null, required: true}},
-  'Android11': {unitTest: {target: 'SL', required: true}, e2e: {target: null, required: true}},
-  'Safari12': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: true}},
-  'Safari13': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: true}},
-  'iOS12': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: true}},
-  'iOS13': {unitTest: {target: 'SL', required: false}, e2e: {target: null, required: true}},
-  'WindowsPhone': {unitTest: {target: 'BS', required: false}, e2e: {target: null, required: true}}
+const config = {
+  'Android10': {unitTest: {target: 'SL', required: true}},
+  'Android11': {unitTest: {target: 'SL', required: true}},
 };
 
-var customLaunchers = {
-  'DartiumWithWebPlatform':
-      {base: 'Dartium', flags: ['--enable-experimental-web-platform-features']},
-  'ChromeNoSandbox': {base: 'Chrome', flags: ['--no-sandbox']},
-  'SL_CHROME': {base: 'SauceLabs', browserName: 'chrome', version: '81'},
-  'SL_CHROMEBETA': {base: 'SauceLabs', browserName: 'chrome', version: 'beta'},
-  'SL_CHROMEDEV': {base: 'SauceLabs', browserName: 'chrome', version: 'dev'},
-  'SL_FIREFOX': {base: 'SauceLabs', browserName: 'firefox', version: '76'},
-  // Firefox 68 is the current ESR vesion
-  'SL_FIREFOXESR': {base: 'SauceLabs', browserName: 'firefox', version: '68'},
-  'SL_FIREFOXBETA':
-      {base: 'SauceLabs', platform: 'Windows 10', browserName: 'firefox', version: 'beta'},
-  'SL_FIREFOXDEV':
-      {base: 'SauceLabs', platform: 'Windows 10', browserName: 'firefox', version: 'dev'},
-  'SL_SAFARI12':
-      {base: 'SauceLabs', browserName: 'safari', platform: 'macOS 10.13', version: '12.1'},
-  'SL_SAFARI13':
-      {base: 'SauceLabs', browserName: 'safari', platform: 'macOS 10.15', version: '13.0'},
-  'SL_IOS12': {
-    base: 'SauceLabs',
-    browserName: 'Safari',
-    platform: 'iOS',
-    version: '12.0',
-    device: 'iPhone 7 Simulator'
-  },
-  'SL_IOS13': {
-    base: 'SauceLabs',
-    browserName: 'Safari',
-    platform: 'iOS',
-    version: '13.0',
-    device: 'iPhone 11 Simulator'
-  },
-  'SL_EDGE': {
-    base: 'SauceLabs',
-    browserName: 'MicrosoftEdge',
-    platform: 'Windows 10',
-    version: '14.14393'
-  },
+/** Whether browsers should be remotely acquired in debug mode. */
+const debugMode = false;
+
+const customLaunchers = {
   'SL_ANDROID10': {
     base: 'SauceLabs',
     browserName: 'Chrome',
-    platform: 'Android',
-    version: '10.0',
-    device: 'Android GoogleAPI Emulator'
+    platformName: 'Android',
+    platformVersion: '10.0',
+    deviceName: 'Google Pixel 3a GoogleAPI Emulator',
+    appiumVersion: '1.20.2',
+    extendedDebugging: debugMode,
   },
   'SL_ANDROID11': {
     base: 'SauceLabs',
     browserName: 'Chrome',
-    platform: 'Android',
-    version: '11.0',
-    device: 'Android GoogleAPI Emulator'
+    platformName: 'Android',
+    platformVersion: '11.0',
+    deviceName: 'Google Pixel 3a GoogleAPI Emulator',
+    appiumVersion: '1.20.2',
+    extendedDebugging: debugMode,
   },
 };
 
-var sauceAliases = {
-  'ALL': Object.keys(customLaunchers).filter(function(item) {
-    return customLaunchers[item].base == 'SauceLabs';
-  }),
-  'DESKTOP': ['SL_CHROME', 'SL_FIREFOX', 'SL_EDGE', 'SL_SAFARI12', 'SL_SAFARI13', 'SL_FIREFOXESR'],
-  'MOBILE': ['SL_ANDROID10', 'SL_ANDROID11', 'SL_IOS12', 'SL_IOS13'],
-  'ANDROID': ['SL_ANDROID10', 'SL_ANDROID11'],
-  'FIREFOX': ['SL_FIREFOXESR'],
-  'IOS': ['SL_IOS12', 'SL_IOS13'],
-  'SAFARI': ['SL_SAFARI12', 'SL_SAFARI13'],
-  'BETA': ['SL_CHROMEBETA', 'SL_FIREFOXBETA'],
-  'DEV': ['SL_CHROMEDEV', 'SL_FIREFOXDEV'],
+const sauceAliases = {
   'CI_REQUIRED': buildConfiguration('unitTest', 'SL', true),
   'CI_OPTIONAL': buildConfiguration('unitTest', 'SL', false)
 };
@@ -112,9 +50,9 @@ module.exports = {
 };
 
 function buildConfiguration(type, target, required) {
-  return Object.keys(CIconfiguration)
+  return Object.keys(config)
       .filter((item) => {
-        var conf = CIconfiguration[item][type];
+        const conf = config[item][type];
         return conf.required === required && conf.target === target;
       })
       .map((item) => target + '_' + item.toUpperCase());

Diff for: karma-js.conf.js

@@ -10,8 +10,6 @@ const browserProvidersConf = require('./browser-providers.conf');
 const {generateSeed} = require('./tools/jasmine-seed-generator');
 const {hostname} = require('os');
 
-// Karma configuration
-// Generated on Thu Sep 25 2014 11:52:02 GMT-0700 (PDT)
 module.exports = function(config) {
   const conf = {
     frameworks: ['jasmine'],
@@ -21,7 +19,7 @@ module.exports = function(config) {
         random: true,
         seed: generateSeed('karma-js.conf'),
       },
-      captureConsole: process.env.CI ? false : true,
+      captureConsole: true,
     },
 
     files: [
@@ -133,8 +131,8 @@ module.exports = function(config) {
       maxDuration: 5400,
     },
 
-    // Try "websocket" for a faster transmission first. Fallback to "polling" if necessary.
-    transports: ['websocket', 'polling'],
+    // Always use `polling` for increased communication stability.
+    transports: ['polling'],
 
     port: 9876,
     captureTimeout: 180000,
@@ -151,6 +149,15 @@ module.exports = function(config) {
     set: () => {},
   });
 
+  if (process.env.CIRCLECI) {
+    conf.frameworks.unshift('parallel');
+    conf.plugins.unshift(require('karma-parallel'));
+    conf.parallelOptions = {
+      executors: 2,
+      shardStrategy: 'round-robin',
+    };
+  }
+
   if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
     console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);
 
@@ -159,6 +166,13 @@ module.exports = function(config) {
     // Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel.
     conf.sauceLabs.build = tunnelIdentifier;
     conf.sauceLabs.tunnelIdentifier = tunnelIdentifier;
+
+    // Patch the `saucelabs` package so that `karma-sauce-launcher` does not attempt downloading
+    // the test logs from upstream and tries re-uploading them with the Karma enhanced details.
+    // This slows-down tests/browser restarting and can decrease stability.
+    // https://github.com/karma-runner/karma-sauce-launcher/blob/59b0c5c877448e064ad56449cd906743721c6b62/src/launcher/launcher.ts#L72-L79.
+    require('saucelabs').default.prototype.downloadJobAsset =
+        () => Promise.resolve('<FAKE-LOGS>');
   }
 
   // For SauceLabs jobs, we set up a domain which resolves to the machine which launched

Diff for: package.json

@@ -176,9 +176,10 @@
     "gulp-conventional-changelog": "^2.0.35",
     "husky": "7.0.4",
     "inquirer": "^8.0.0",
+    "karma-parallel": "^0.3.1",
     "karma-sauce-launcher": "^4.3.6",
     "madge": "^5.0.0",
-    "sauce-connect": "https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz",
+    "sauce-connect": "https://saucelabs.com/downloads/sc-4.7.1-linux.tar.gz",
     "semver": "^7.3.5",
     "ts-node": "^10.0.0",
     "tsec": "0.2.0",
@@ -192,6 +193,7 @@
   "// 5": "Ensure a single version of webdriver-manager so it is hoisted as the integration tests depend on it being found at ../../node_modules/webdriver-manager",
   "// 6": "Ensure that `@babel/*` packages match the below versions to avoid conflicts with `types/babel__*`",
   "// 7": "Ensure that transitive dependencies on `https-proxy-agent` are at minimum v5 as older versions patch NodeJS directly, breaking tools like webdriver which is used by the karma-sauce-launcher as an example.",
+  "// 8": "Ensure that a single instance of the `saucelabs` package is used. Protractor and the Karma sauce launcher pull this package as dependency. A single instance allows for e.g. easier patching in the Karma config.",
   "resolutions": {
     "**/graceful-fs": "4.2.8",
     "**/webdriver-manager": "12.1.8",
@@ -202,6 +204,7 @@
     "@babel/template": "7.8.6",
     "@babel/traverse": "7.8.6",
     "@babel/types": "7.8.6",
-    "**/https-proxy-agent": "5.0.0"
+    "**/https-proxy-agent": "5.0.0",
+    "**/saucelabs": "4.7.8"
   }
 }