Add Sauce RDC and EmuSim cron jobs (#711)

* Add Sauce emusim staging tests * Get the appium staging url for bintray * Use daily sliding window to determine devices to test * Make web tests use remote URL * Also remove driver tests because they are internal tests * Add RDC tests * Add web testing * Use mocha parallel testing * Travis * Travis * Travis * Travis * Cron job only * Fixes * Fixes * WDA test fixes * PR fixes * PR fix * PR fix * Dynamic device allocation for RDC * Add documentation * Update .app builds
appium · Jul 10, 2018 · dac9a79 · dac9a79
1 parent ce8785d
commit dac9a79
Show file tree

Hide file tree

Showing 21 changed files with 377 additions and 50 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,44 +10,126 @@ env:
 matrix:
   include:
     - osx_image: xcode9.3
+      if: type != cron
       env: TEST=unit COVERALLS=1 _FORCE_LOGS=0
 
     - osx_image: xcode9.3
+      if: type != cron
       env: CI_METRICS=1 TEST=functional/basic
 
     - osx_image: xcode9.3
+      if: type != cron
       env: TEST=functional/driver
 
     - osx_image: xcode9.3
+      if: type != cron
       env: CI_METRICS=1 TEST=functional/web
 
     - osx_image: xcode9.3
+      if: type != cron
       env: CI_METRICS=1 TEST=functional/long
 
     # smoke test on xcode 9.3/iOS 11.2 web
     - osx_image: xcode9.3
+      if: type != cron
       env: CI_METRICS=1 PLATFORM_VERSION=11.2 TEST=functional/web
 
     # smoke test on xcode 8.3 web
     - osx_image: xcode8.3
+      if: type != cron
       env: CI_METRICS=1 PLATFORM_VERSION=10.3 TEST=functional/web
 
     # smoke test on xcode 9.2 web
     - osx_image: xcode9.2
       env: CI_METRICS=1 PLATFORM_VERSION=11.2 TEST=functional/web
+
+    # smoke test on xcode 9.3 web
+    - osx_image: xcode9.2
+      if: type != cron
+      env: CI_METRICS=1 PLATFORM_VERSION=11.2 TEST=functional/web
+
+    # Sauce Emusim tests
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=0 TEST=functional/basic
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=0 TEST=functional/web
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=0 TEST=functional/long
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=1 TEST=functional/basic
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=1 TEST=functional/web
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=1 TEST=functional/long
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=2 TEST=functional/basic
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=2 TEST=functional/web
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=2 TEST=functional/long
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=3 TEST=functional/basic
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=3 TEST=functional/web
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=3 TEST=functional/long
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=4 TEST=functional/basic
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=4 TEST=functional/web
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_EMUSIM_DEVICE_INDEX=4 TEST=functional/long
+
+    # Sauce RDC Tests
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_RDC_DEVICE_INDEX=0 TEST=functional/basic
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_RDC_DEVICE_INDEX=0 TEST=functional/web
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_RDC_DEVICE_INDEX=0 TEST=functional/long
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_RDC_DEVICE_INDEX=1 TEST=functional/basic
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_RDC_DEVICE_INDEX=1 TEST=functional/web
+    - os: linux
+      if: type = cron
+      env: CLOUD=1 SAUCE_RDC_DEVICE_INDEX=1 TEST=functional/long
+
 git:
   submodules: false
 before_install:
   # Use sed to replace the SSH URL with the public URL, then initialize submodules
   # code from http://stackoverflow.com/a/24600210/375688
-  - sed -i '' 's/git@github.com:/https:\/\/github.com\//' /Users/travis/build/appium/appium-xcuitest-driver/.gitmodules
-  - git submodule update --init --recursive
+  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+      sed -i '' 's/git@github.com:/https:\/\/github.com\//' /Users/travis/build/appium/appium-xcuitest-driver/.gitmodules;
+      git submodule update --init --recursive;
+    fi
 script:
-  - npm run lint && npm run mocha -- -t 480000 --recursive build/test/$TEST -g @skip-ci -i --exit
+  - npm run lint && npm run mocha -- -t 480000 --require build/test/env/env --recursive build/test/$TEST -g @skip-ci -i --exit
   - if [ -n "$COVERALLS" ]; then npm run coverage; fi
   - if [ -n "$CI_METRICS" ]; then
       mkdir -p ./ci-metrics && ls -la ./ci-metrics;
       nvm install 7;
       npm install -g appium-event-parser;
       appium-event-parser -s -i ./ci-metrics;
-    fi
+    fi
diff --git a/docs/cloud-testing.md b/docs/cloud-testing.md
@@ -0,0 +1,25 @@
+# Cloud Testing
+
+With a little bit of configuration, the E2E tests can be run on SauceLabs Real Device Cloud and on Simulators.
+
+## Environment Variables
+* Running tests on Sauce Real Device Cloud (RDC) or Sauce OnDemand Simulators requires a SauceLabs username and access key. 
+
+### Real Device Cloud
+* Refer to [real env file](/test/env/env-ios-real.js) to see which environment variables need to be set to access the RDC Cloud
+* To run tests locally, must set two environment variables   
+  * `CLOUD_PLATFORM_VERSION` A supported [iOS version](https://saucelabs.com/devices). No need to set PLATFORM_VERSION. A device will be dynamically allocated
+  * `SAUCE_RDC` Needs to be set to `true`
+
+### Simulator Cloud
+* Refer to [sim env file](/test/env/env-ios-real.js) to see which environment variables need to be set for SauceLabs OnDemand Simulator testing
+* To run tests locally, must set three environment variables
+  * `CLOUD_PLATFORM_VERSION` A supported [iOS version](https://saucelabs.com/platforms)
+  * `CLOUD_DEVICE_NAME` A supported [iOS device](https://saucelabs.com/platforms) that corresponds to that OS version
+  * `SAUCE_EMUSIM` Needs to be set to true
+
+## Running the Tests
+Run the tests with the following command:
+
+```gulp transpile && mocha mocha  --require build/test/env/env --timeout 4000000 --recursive build/<TEST_PATH>```
+
diff --git a/package.json b/package.json
@@ -81,17 +81,21 @@
     "eslint-plugin-promise": "^3.3.1",
     "glob": "^7.1.0",
     "gulp": "^3.8.11",
-    "ios-test-app": "^2.5.7",
-    "ios-uicatalog": "^1.0.4",
     "mocha": "^5.1.1",
+    "moment": "^2.22.2",
     "pem": "^1.8.3",
     "pngjs": "^3.3.1",
     "pre-commit": "^1.1.3",
     "sinon": "^6.0.0",
+    "sync-request": "^6.0.0",
     "through2": "^2.0.0",
     "unzip": "^0.1.11",
     "wd": "^1.5.0"
   },
+  "optionalDependencies": {
+    "ios-test-app": "^2.5.7",
+    "ios-uicatalog": "^1.0.4"
+  },
   "greenkeeper": {
     "ignore": [
       "babel-eslint",

diff --git a/test/env/env-base.js b/test/env/env-base.js
@@ -0,0 +1,3 @@
+export default {
+  CLOUD: true,
+};
diff --git a/test/env/env-ios-real.js b/test/env/env-ios-real.js
@@ -0,0 +1,30 @@
+import moment from 'moment';
+import envBase from './env-base';
+import { logger } from 'appium-support';
+
+const platforms = [
+  '10',
+  '11',
+  '10.3',
+  '11.3',
+  '11.4'
+];
+
+const configIndex = process.env.SAUCE_RDC_DEVICE_INDEX || 0;
+
+// Get the two platform versions to use based on a sliding window
+const platformIndex = (moment().dayOfYear() * 2) % platforms.length + (parseInt(configIndex, 0) || 0);
+
+const CLOUD_PLATFORM_VERSION = process.env.CLOUD_PLATFORM_VERSION || platforms[platformIndex % platforms.length];
+
+logger.getLogger('CI STAGING TESTS').info(`Running tests on real, dynamically allocated device with iOS version: ${CLOUD_PLATFORM_VERSION}`);
+
+export default {
+  ...envBase,
+  REAL_DEVICE: 1,
+  SAUCE_RDC: true,
+  SAUCE_RDC_USERNAME: process.env.TESTOBJECT_USERNAME || process.env.SAUCE_USERNAME,
+  SAUCE_RDC_ACCESS_KEY: process.env.TESTOBJECT_API_KEY,
+  SAUCE_RDC_WEB_ACCESS_KEY: process.env.TESTOBJECT_WEB_API_KEY,
+  CLOUD_PLATFORM_VERSION,
+};
diff --git a/test/env/env-ios-sim.js b/test/env/env-ios-sim.js
@@ -0,0 +1,40 @@
+import moment from 'moment';
+import _ from 'lodash';
+import envBase from './env-base';
+import { logger } from 'appium-support';
+import platformDefinition from './ios-sim-platforms';
+
+const platforms = [];
+
+let deviceNameIndex = 0;
+let devicesRemaining;
+
+// Get a list of tuples containing [PLATFORM_VERSION, DEVICE_NAME]
+do {
+  devicesRemaining = false;
+
+  // Do a transpose of all of the device arrays so that we don't get the same
+  // OS grouped together
+  for (let [platformVersion, devices] of _.toPairs(platformDefinition)) {
+    if (devices.length > deviceNameIndex) {
+      platforms.push([platformVersion, devices[deviceNameIndex]]);
+      devicesRemaining = true;
+    }
+  }
+  deviceNameIndex++;
+} while (devicesRemaining);
+
+// Get the device based on a 5-day sliding window
+const platformIndex = (moment().dayOfYear() * 5) % platforms.length + (parseInt(process.env.SAUCE_EMUSIM_DEVICE_INDEX, 0) || 0);
+const [CLOUD_PLATFORM_VERSION, CLOUD_DEVICE_NAME] = platforms[platformIndex % platforms.length];
+
+logger.getLogger('CI STAGING TESTS').info(`Running tests on iOS ${CLOUD_PLATFORM_VERSION}, device "${CLOUD_DEVICE_NAME}"`);
+
+export default {
+  ...envBase,
+  SAUCE_EMUSIM: true,
+  SAUCE_USERNAME: process.env.SAUCE_USERNAME,
+  SAUCE_ACCESS_KEY: process.env.SAUCE_ACCESS_KEY,
+  CLOUD_PLATFORM_VERSION: process.env.CLOUD_PLATFORM_VERSION || CLOUD_PLATFORM_VERSION,
+  CLOUD_DEVICE_NAME: process.env.CLOUD_DEVICE_NAME || CLOUD_DEVICE_NAME,
+};
diff --git a/test/env/env.js b/test/env/env.js
@@ -0,0 +1,31 @@
+import _ from 'lodash';
+import request from 'sync-request';
+import { logger } from 'appium-support';
+
+const log = logger.getLogger('CI STAGING TESTS');
+
+let env = {};
+
+
+// Get the environment variables
+if (!_.isEmpty(process.env.SAUCE_EMUSIM_DEVICE_INDEX) || process.env.SAUCE_EMUSIM) {
+  log.info('Running tests on SauceLabs OnDemand');
+  Object.assign(env, require('./env-ios-sim'));
+} else if (!_.isEmpty(process.env.SAUCE_RDC_DEVICE_INDEX) || process.env.SAUCE_RDC) {
+  log.info('Running tests on SauceLabs real device cloud');
+  Object.assign(env, require('./env-ios-real'));
+}
+
+if (env.CLOUD) {
+  // Find the latest bundle
+  log.info('Getting the sha of the most recent master commit');
+  const res = request('GET', 'https://api.bintray.com/packages/appium-builds/appium/appium/files', {json: true});
+  const {name:bundleName} = JSON.parse(res.getBody('utf8'))[0];
+
+  // Get the URL
+  const stagingUrl = `https://bintray.com/appium-builds/appium/download_file?file_path=${bundleName}`;
+  log.info(`Using staging URL: ${stagingUrl}`);
+  env.APPIUM_STAGING_URL = stagingUrl;
+}
+
+Object.assign(process.env, env);
diff --git a/test/env/ios-sim-platforms.js b/test/env/ios-sim-platforms.js
@@ -0,0 +1,30 @@
+export default {
+  '11.2': [
+    'iPhone 5s Simulator', 'iPhone 6s Simulator', 'iPhone 7 Simulator', 'iPhone 8 Plus Simulator',
+    'iPhone X Simulator', 'iPad Air 2 Simulator', 'iPad Pro (9.7 inch) Simulator'
+  ],
+  '11.1': [
+    'iPhone 5s Simulator', 'iPhone 6s Simulator', 'iPhone 7 Simulator', 'iPhone 8 Plus Simulator',
+    'iPhone X Simulator', 'iPad Air 2 Simulator', 'iPad Pro (9.7 inch) Simulator'
+  ],
+  '11': [
+    'iPhone 5s Simulator', 'iPhone 6s Plus Simulator', 'iPhone 7 Simulator', 'iPhone X Simulator',
+    'iPad Air 2 Simulator', 'iPad iPad Pro (12.9 inch) Simulator', 'iPad Pro (12.9 inch) (2nd generation) Simulator'
+  ],
+  '10.3': [
+    'iPhone 5s Simulator', 'iPhone 6 Plus Simulator', 'iPhone 6 Plus Simulator', 'iPhone 6s Simulator',
+    'iPhone SE Simulator', 'iPad Air 2 Simulator', 'iPad Pro (9.7 inch) Simulator'
+  ],
+  '10.2': [
+    'iPhone 6s Plus Simulator', 'iPhone SE Simulator', 'iPhone 6s Simulator', 'iPhone 7 Plus Simulator',
+    'iPad Pro (9.7 inch) Simulator', 'iPad Simulator', 'iPad Simulator'
+  ],
+  '10': [
+    'iPhone 5 Simulator', 'iPhone 6 Plus Simulator', 'iPhone 6s Simulator', 'iPhone 7 Simulator',
+    'iPad Air 2 Simulator', 'iPad Retina Simulator', 'iPad Simulator'
+  ],
+  '9.3': [
+    'iPhone 4s Simulator', 'iPhone 6 Plus Simulator', 'iPhone 6s Simulator', 'iPhone 6 Plus Simulator',
+    'iPad Air 2 Simulator', 'iPad Retina Simulator', 'iPad Simulator'
+  ],
+};
diff --git a/test/functional/apps.js b/test/functional/apps.js
@@ -0,0 +1,36 @@
+import path from 'path';
+import { system } from 'appium-support';
+
+let testAppPath, uiCatalogApp;
+
+// Had to make these two optional dependencies so the tests
+// can still run in linux
+if (system.isMac()) {
+  testAppPath = require('ios-test-app').absolute;
+  uiCatalogApp = require('ios-uicatalog');
+}
+
+const apps = {};
+
+const {REAL_DEVICE, CLOUD} = process.env;
+
+if (REAL_DEVICE) {
+  if (CLOUD) {
+    apps.testAppId = 1;
+  } else {
+    apps.iosTestApp = testAppPath.iphoneos;
+    apps.uiCatalogApp = path.resolve('.', 'node_modules', 'ios-uicatalog', uiCatalogApp[0]);
+  }
+} else {
+  if (CLOUD) {
+    apps.iosTestApp = 'http://appium.github.io/appium/assets/TestApp9.4.app.zip';
+    apps.uiCatalogApp = 'http://appium.github.io/appium/assets/UICatalog9.4.app.zip';
+    apps.touchIdApp = null; // TODO: Upload this to appium.io
+  } else {
+    apps.iosTestApp = testAppPath.iphonesimulator;
+    apps.uiCatalogApp = path.resolve('.', 'node_modules', 'ios-uicatalog', uiCatalogApp[1]);
+    apps.touchIdApp = path.resolve('.', 'test', 'assets', 'TouchIDExample.app');
+  }
+}
+
+export default apps;
diff --git a/test/functional/basic/alert-e2e-specs.js b/test/functional/basic/alert-e2e-specs.js
@@ -39,7 +39,7 @@ describe('XCUITestDriver - alerts -', function () {
   it('should detect Simple', async function () {
     let el = await driver.elementByAccessibilityId('Simple');
     await el.click();
-    await B.delay(2000);
+    await B.delay(process.env.CLOUD ? 10000 : 2000);
 
     (await driver.alertText()).should.include('A Short Title Is Best');
     await driver.dismissAlert();

diff --git a/test/functional/basic/basic-e2e-specs.js b/test/functional/basic/basic-e2e-specs.js
@@ -25,10 +25,19 @@ describe('XCUITestDriver - basics -', function () {
   describe('status -', function () {
     it('should get the server status', async function () {
       let status = await driver.status();
-      status.wda.should.exist;
+      if (process.env.SAUCE_EMUSIM) {
+        status.build.version.should.equal('Sauce Labs');
+      } else {
+        status.wda.should.exist;
+      }
     });
 
     it('should return status immediately if another operation is in progress', async function () {
+      // Sauce EmuSim/RDC don't seem to support getting status and running an operation concurrently
+      if (process.env.CLOUD) {
+        this.skip();
+      }
+
       await driver.setImplicitWaitTimeout(10000);
       let findElementPromise = B.resolve(driver.elementById('WrongLocator'));
       let status = await driver.status();