Fix: Functional tests + cleanup annotations after Features

.eslintrc

@@ -26,7 +26,7 @@
         "Assert": false,
         "Feature": false,
         "Before": false,
-        "BeforeSuite": false,
+        "After": false,
         "Scenario": false
     },
     "rules": {

.travis.yml

@@ -15,16 +15,13 @@ aliases:
         - sleep 5
         addons:
           sauce_connect: true
-        script: yarn run functional-tests-ci
-    - &sauce-labs-mobile
-      <<: *sauce-labs
-      script: yarn run functional-tests-ci --grep @mobile
+        script: travis_retry yarn run functional-tests-ci
     - &sauce-labs-desktop
       <<: *sauce-labs
-      script: yarn run functional-tests-ci --grep @desktop
+      script: travis_retry yarn run functional-tests-ci --grep @desktop
     - &sauce-labs-ie
       <<: *sauce-labs
-      script: yarn run functional-tests-ci --grep @ie
+      script: travis_retry yarn run functional-tests-ci --grep @ie
 jobs:
   include:
   - script: yarn run ci
@@ -42,15 +39,4 @@ jobs:
     env: BROWSER_PLATFORM="Windows 10" BROWSER_NAME="MicrosoftEdge" FILE_ID="285567976309" FILE_VERSION_ID="300496707445"
   # Windows IE
   - <<: *sauce-labs-ie
-    env: BROWSER_PLATFORM="Windows 10" BROWSER_NAME="internet explorer" FILE_ID="285568624824" FILE_VERSION_ID="300497342136"
-  # iPhone
-  - <<: *sauce-labs-mobile
-    env: BROWSER_PLATFORM="iOS" DEVICE_NAME="iPhone 6 Simulator" PLATFORM_VERSION="11.2" BROWSER_NAME="Safari" FILE_ID="285569765346" FILE_VERSION_ID="300498497346"
-  # iPad
-  - <<: *sauce-labs-mobile
-    # Uses firefox file id
-    env: BROWSER_PLATFORM="iOS" DEVICE_NAME="iPad Simulator" PLATFORM_VERSION="11.2" BROWSER_NAME="Safari" FILE_ID="285568802145" FILE_VERSION_ID="300497533713"
-  # Android
-  - <<: *sauce-labs-mobile
-    # Uses chrome file id
-    env: BROWSER_PLATFORM="Android" DEVICE_NAME="Android GoogleAPI Emulator" PLATFORM_VERSION="7.1" BROWSER_NAME="Chrome" FILE_ID="285567874839" FILE_VERSION_ID="300496591287"
+    env: BROWSER_PLATFORM="Windows 10" BROWSER_NAME="internet explorer" FILE_ID="285568624824" FILE_VERSION_ID="300497342136"

codecept.conf.js

@@ -8,9 +8,11 @@ const {
     BROWSER_PLATFORM,
     PLATFORM_VERSION,
     DEVICE_NAME,
-    DEFAULT_WAIT_TIME = 90000
+    DEFAULT_WAIT_TIME = 90000,
+    RUN_LOCALLY=false
 } = process.env;
 const MOBILE_PLATFORMS = ['iOS', 'Android'];
+const { cleanupAnnotations } = require('./functional-tests/helpers/cleanup');
 
 // Local selenium config
 const commonConfigObj = {
@@ -21,7 +23,7 @@ const commonConfigObj = {
 };
 
 const helperObj = {};
-const isLocalBuild = typeof SAUCE_USERNAME === 'undefined';
+const isLocalBuild = typeof SAUCE_USERNAME === 'undefined' || RUN_LOCALLY;
 
 if (isLocalBuild) {
     helperObj.WebDriverIO = commonConfigObj;
@@ -61,14 +63,21 @@ if (isLocalBuild) {
     }
 }
 
+/**
+ * @return {void}
+*/
+function cleanup() {
+    cleanupAnnotations() ;
+}
+
 exports.config = {
     tests: './functional-tests/tests/*.js',
     timeout: DEFAULT_WAIT_TIME,
     output: './functional-tests/output',
     helpers: helperObj,
     include: {},
-    bootstrap: './functional-tests/helpers/cleanup.js',
-    teardown: './functional-tests/helpers/cleanup.js',
+    bootstrap: cleanup,
+    teardown: cleanup,
     mocha: {},
     name: 'box-annotations',
     hooks: isLocalBuild ? [] : ['./functional-tests/helpers/eventHooks.js']

functional-tests/helpers/actions.js

@@ -10,7 +10,15 @@ const {
 } = require('../helpers/constants');
 const { validateReply, validateDeleteConfirmation } = require('./validation');
 
-function replyToThread(I) {
+/**
+ * Replies to an annotation thread
+ *
+ * @param {Object} I - the codeceptjs I
+ * @param {string} selector - annotation CSS selector
+ *
+ * @return {void}
+ */
+function replyToThread(I, selector) {
     I.say('Reply to highlight comment annotation');
     I.fillField(SELECTOR_REPLY_TEXTAREA, 'Sample reply');
     I.click(`${SELECTOR_REPLY_CONTAINER} ${SELECTOR_ANNOTATION_BUTTON_POST}`);
@@ -20,15 +28,25 @@ function replyToThread(I) {
     I.say('Cancel a reply to a highlight comment annotation');
     I.fillField(SELECTOR_REPLY_TEXTAREA, 'Sample canceled reply');
     I.click(`${SELECTOR_REPLY_CONTAINER} ${SELECTOR_ANNOTATION_BUTTON_CANCEL}`);
-    I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 2);
+    I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 1);
 }
 
-function deleteAnnotation(I, annotationCount) {
+/**
+ * Replies to an annotation thread
+ *
+ * @param {Object} I - the codeceptjs I
+ * @param {number} annotationCount - current number of annotations in threads
+ * @param {string} selector - the selector to use, defaults to .annotation-comment
+ *
+ * @return {void}
+ */
+function deleteAnnotation(I, annotationCount, selector = SELECTOR_ANNOTATION_COMMENT) {
     I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, annotationCount);
 
     I.say('Delete the annotation');
-    I.click(SELECTOR_DELETE_COMMENT_BTN);
-    validateDeleteConfirmation(I);
+    I.waitForEnabled(`${selector} ${SELECTOR_DELETE_COMMENT_BTN}`, 9);
+    I.click(`${selector} ${SELECTOR_DELETE_COMMENT_BTN}`);
+    validateDeleteConfirmation(I, selector);
 
     I.say('Annotation should be deleted');
     if (annotationCount > 1) {

functional-tests/helpers/cleanup.js

@@ -25,7 +25,7 @@ function deleteAnnotation(annotation) {
     });
 }
 
-module.exports = function() {
+function cleanupAnnotations() {
     client.get(`/files/${FILE_ID}/annotations?version=${FILE_VERSION_ID}`, {}, function(err, response) {
         if (err) {
             // handle error
@@ -41,4 +41,6 @@ module.exports = function() {
         console.log(`Deleting ${entries.length} annotations`);
         entries.forEach(deleteAnnotation);
     });
-}
+}
+
+exports.cleanupAnnotations = cleanupAnnotations;

functional-tests/helpers/validation.js

@@ -18,12 +18,32 @@ const {
 } = require('../helpers/constants');
 const { expect } = require('chai');
 
+
+/**
+ * Ensures the SVG icon is of the expected color
+ *
+ * @param {Object} I - the codeceptjs I
+ * @param {string} selector - the selector to use
+ * @param {string} color - rgb icon color
+ *
+ * @return {void}
+ */
 async function validateIconColor(I, selector, color) {
     I.waitForElement(selector);
     const clr = await I.grabCssPropertyFrom(`${selector} svg`, 'fill');
     expect(clr).to.equal(color);
 }
 
+
+/**
+ * Validates that the text area appears as expected
+ *
+ * @param {Object} I - the codeceptjs I
+ * @param {string} containerSel - the container selector to use
+ * @param {string} textareaSel - the textarea selector to use
+ *
+ * @return {void}
+ */
 function* validateTextarea(I, containerSel, textareaSel) {
     I.say(`Validate ${containerSel} ${textareaSel}`);
     I.waitForVisible(`${containerSel} ${textareaSel}${SELECTOR_ACTIVE}`);
@@ -39,19 +59,31 @@ function* validateTextarea(I, containerSel, textareaSel) {
     I.waitForVisible(SELECTOR_ANNOTATION_DIALOG);
 }
 
+/**
+ * Validates that the annotation appears as expected
+ *
+ * @param {Object} I - the codeceptjs I
+ *
+ * @return {void}
+ */
 function validateAnnotation(I) {
     I.say('Dialog should contain new annotation');
     I.waitForVisible(SELECTOR_ANNOTATION_DIALOG);
-    I.see('Posting...', SELECTOR_USER_NAME);
     I.waitForVisible(SELECTOR_ANNOTATION_CONTAINER);
     I.waitNumberOfVisibleElements(SELECTOR_ANNOTATION_COMMENT, 1);
     I.waitForEnabled(SELECTOR_DELETE_COMMENT_BTN);
     I.waitForVisible(SELECTOR_PROFILE_IMG_CONTAINER);
-    I.waitForText('Kanye West', 10, SELECTOR_USER_NAME);
-
     validateTextarea(I, SELECTOR_REPLY_CONTAINER, SELECTOR_REPLY_TEXTAREA);
+    I.waitForText('Kanye West', 15, `${SELECTOR_ANNOTATION_COMMENT} ${SELECTOR_USER_NAME}`);
 }
 
+/**
+ * Validates that the annotation reply appears as expected
+ *
+ * @param {Object} I - the codeceptjs I
+ *
+ * @return {void}
+ */
 function validateReply(I) {
     I.say('Reply should be added to dialog');
     I.waitForVisible(SELECTOR_ANNOTATION_DIALOG);
@@ -64,22 +96,31 @@ function validateReply(I) {
     validateTextarea(I, SELECTOR_REPLY_CONTAINER, SELECTOR_REPLY_TEXTAREA);
 }
 
-function validateDeleteConfirmation(I) {
+/**
+ * Validates that the delete confirmation message appears
+ * and acts as expected
+ *
+ * @param {Object} I - the codeceptjs I
+ * @param {string} selector - the selector to use
+ *
+ * @return {void}
+ */
+function validateDeleteConfirmation(I, selector = '') {
     I.say('Validate delete confirmation');
-    I.waitForText('Delete this annotation?', 10, SELECTOR_DELETE_CONFIRM_MESSAGE);
-    I.waitForVisible(SELECTOR_CONFIRM_DELETE_BTN);
-    I.waitForVisible(SELECTOR_CANCEL_DELETE_BTN);
+    I.waitForText('Delete this annotation?', 10, `${selector} ${SELECTOR_DELETE_CONFIRM_MESSAGE}`);
+    I.waitForVisible(`${selector} ${SELECTOR_CONFIRM_DELETE_BTN}`);
+    I.waitForVisible(`${selector} ${SELECTOR_CANCEL_DELETE_BTN}`);
 
     // Cancel annotation delete
-    I.click(SELECTOR_CANCEL_DELETE_BTN);
-    I.waitForInvisible(SELECTOR_DELETE_CONFIRM_MESSAGE);
+    I.click(`${selector} ${SELECTOR_CANCEL_DELETE_BTN}`);
+    I.waitForInvisible(`${selector} ${SELECTOR_DELETE_CONFIRM_MESSAGE}`);
 
     // Delete the annotation
-    I.click(SELECTOR_DELETE_COMMENT_BTN);
+    I.click(`${selector} ${SELECTOR_DELETE_COMMENT_BTN}`);
 
     // Delete confirmation should appear
-    I.waitForVisible(SELECTOR_CONFIRM_DELETE_BTN);
-    I.click(SELECTOR_CONFIRM_DELETE_BTN);
+    I.waitForVisible(`${selector} ${SELECTOR_CONFIRM_DELETE_BTN}`);
+    I.click(`${selector} ${SELECTOR_CONFIRM_DELETE_BTN}`);
 }
 
 exports.validateIconColor = validateIconColor;

functional-tests/run-all.js

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+/* eslint-disable no-console */
+const async = require('async');
+const util = require('util');
+const colors = require('colors');
+const exec = util.promisify(require('child_process').exec);
+
+const { SAUCE_USERNAME, SAUCE_ACCESS_KEY, TRAVIS_JOB_NUMBER } = process.env;
+
+// browsers
+const CHROME = 'chrome';
+const FIREFOX = 'firefox';
+const EDGE = 'MicrosoftEdge';
+const IE = 'internet explorer';
+
+// platforms
+const SAFARI = 'Safari';
+const WINDOWS = 'Windows 10';
+const OSX = 'macOS 10.13';
+const ios = 'iOS';
+const android = 'Android';
+
+// file information
+const CHROME_FILE = {
+    id: '285567874839',
+    version: '300496591287'
+};
+const SAFARI_FILE = {
+    id: '285569765346',
+    version: '300498497346'
+};
+const FIREFOX_FILE = {
+    id: '285568802145',
+    version: '300497533713'
+};
+const EDGE_FILE = {
+    id: '285567976309',
+    version: '300496707445'
+};
+const IE_FILE = {
+    id: '285568624824',
+    version: '300497342136'
+};
+
+const envArr = [
+    `BROWSER_PLATFORM="${OSX}" BROWSER_NAME="${CHROME}" FILE_ID="${CHROME_FILE.id}" FILE_VERSION_ID="${CHROME_FILE.version}"`,
+    `BROWSER_PLATFORM="${OSX}" BROWSER_NAME="safari" FILE_ID="${SAFARI_FILE.id}" FILE_VERSION_ID="${SAFARI_FILE.version}"`,
+    `BROWSER_PLATFORM="${OSX}" BROWSER_NAME="${FIREFOX}" FILE_ID="${FIREFOX_FILE.id}" FILE_VERSION_ID="${FIREFOX_FILE.version}"`,
+    `BROWSER_PLATFORM="${WINDOWS}" BROWSER_NAME="${EDGE}" FILE_ID="${EDGE_FILE.id}" FILE_VERSION_ID="${EDGE_FILE.version}"`
+];
+
+if (!TRAVIS_JOB_NUMBER || !SAUCE_USERNAME || !TRAVIS_JOB_NUMBER) {
+    throw new Error('missing TRAVIS_JOB_NUMBER, SAUCE_USERNAME, or TRAVIS_JOB_NUMBER');
+}
+
+const processArr = [];
+async.eachLimit(
+    envArr,
+    4,
+    async (envStr) => {
+        let grepStr = '';
+
+        const mobileRegex = /iOS|Android/;
+
+        if (mobileRegex.test(envStr)) {
+            grepStr = '--grep "@mobile"';
+        } else {
+            grepStr = '--grep "@desktop"';
+        }
+
+        const cmd = `cd .. && CI=true SAUCE_USERNAME=${SAUCE_USERNAME} SAUCE_ACCESS_KEY=${SAUCE_ACCESS_KEY} TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER} ${envStr} node ./node_modules/codeceptjs/bin/codecept.js run --steps ${grepStr}`;
+
+        console.log('Running cmd: ', cmd);
+        const process = exec(cmd);
+        processArr.push(process);
+        await process;
+    },
+    (err) => {
+        if (err) {
+            console.log(colors.red.underline(err));
+            console.log(colors.red(err.stdout));
+            processArr.forEach((process) => {
+                if (process && process.kill) {
+                    try {
+                        process.kill();
+                    } catch (err2) {
+                        console.error(err2);
+                    }
+                }
+            });
+            throw new Error();
+        }
+        console.log('SUCCESS!');
+    }
+);

functional-tests/tests/draw.js

@@ -17,13 +17,18 @@ const {
 } = require('../helpers/constants');
 
 const { draw, clickAtLocation } = require('../helpers/mouseEvents');
+const { cleanupAnnotations } = require('../helpers/cleanup');
 
 Feature('Draw Annotation Sanity');
 
 Before(function(I) {
     I.amOnPage('/');
 });
 
+After(function() {
+    cleanupAnnotations();
+});
+
 Scenario('Create/Delete drawing @desktop', function(I) {
     /*
      * Can enter/exit drawing mode properly @desktop