More test improvements (#701)

* Adding more functional test improvements

* Update readme

* Wait for status bar before clicking collection

* Fix clicking create index button

* Better index input checks

* Adding additional wait for status bar

test/README.md

@@ -75,10 +75,10 @@ describe('Compass Functional Test Suite #spectron', function() {
 
 ### Commands
 
-- [Wait Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L44)
-- [Click Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L232)
-- [Get Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L545)
-- [Input Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L757)
+- [Wait Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L156)
+- [Click Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L353)
+- [Get Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L735)
+- [Input Commands](https://github.com/10gen/compass/blob/master/test/support/spectron-support.js#L1079)
 
 ## Tips
 
@@ -104,11 +104,6 @@ loading the application without any erros anywhere.
 
 ### Timeouts
 
-Most commands in the suite use a default timeout of 10 seconds, set as the
-`TIMEOUT` constant. Operations that take longer use the `LONG_TIMEOUT` constant
-which is 30 seconds. Be careful to note, however, to not use a timeout if at
-all possible. Excessive timeouts within the test can cause the entire block
-to fail if they take too long, resulting in not knowing exactly which command
-in the test failed. As rule rule it's better to fail fast and adjust accordingly,
-or to change the "wait" part of the test to find something appropriate to wait
-on faster.
+Please use the custom methods `waitForVisibleInCompass`, `waitForExistsInCompass`
+and `waitUntilInCompass` to leverage the incremental timeout functionality to
+provide faster and more stable tests.

test/compass-functional.test.js

@@ -371,6 +371,7 @@ describe('Compass Functional Test Suite #spectron', function() {
 
           it('removes the collection from the sidebar', function() {
             return client
+              .waitForInstanceRefresh()
               .getSidebarCollectionNames()
               .should.not.eventually.include('music.labels');
           });
@@ -487,12 +488,15 @@ describe('Compass Functional Test Suite #spectron', function() {
 
         it('applies the filter again while on schema tab', function() {
           return client
-          .clickCollectionInSidebar('music.artists')
-          .waitForStatusBar()
-          .inputFilterFromSchemaTab(filter)
-          .clickApplyFilterButtonFromSchemaTab()
-          .getSamplingMessageFromSchemaTab()
-          .should.eventually
+            .waitForStatusBar()
+            .clickCollectionInSidebar('music.artists')
+            .waitForStatusBar()
+            .inputFilterFromSchemaTab(filter)
+            .waitForStatusBar()
+            .clickApplyFilterButtonFromSchemaTab()
+            .getSamplingMessageFromSchemaTab()
+            .should
+            .eventually
             .equal('Query returned 0 documents. This report is based on a sample of 0 documents (0.00%).');
         });
 

test/support/spectron-support.js

@@ -84,14 +84,34 @@ function progressiveWait(fn, selector, reverse, index) {
   debug(`Looking for element ${selector} with timeout ${timeout}ms`);
   return fn(selector, timeout, reverse)
     .catch(function(e) {
-      if (isTimeoutError(e) && timeout !== 21000) {
+      if (isTimeoutError(e) && timeout !== 13000) {
         return progressiveWait(fn, selector, reverse || false, index + 1);
       } else {
         throw e;
       }
     });
 }
 
+/**
+ * Waits until the provided funciton returns true.
+ *
+ * @param {Function} waitUntil - The waitUntil function.
+ * @param {Function} fn - The function to execute.
+ * @param {Number} index - The timeout index.
+ */
+function progressiveWaitUntil(waitUntil, fn, index) {
+  const timeout = TIMEOUTS[index];
+  debug(`Waiting until function returns with timeout ${timeout}ms`);
+  return waitUntil(fn, timeout)
+    .catch(function(e) {
+      if (isTimeoutError(e) && timeout !== 13000) {
+        return progressiveWaitUntil(waitUntil, fn, index + 1);
+      } else {
+        throw e;
+      }
+    });
+}
+
 /**
  * Add the extended wait commands for Compass.
  */
@@ -116,6 +136,15 @@ function addExtendedWaitCommands(client) {
   client.addCommand('waitForVisibleInCompass', function(selector, reverse) {
     return progressiveWait(this.waitForVisible.bind(this), selector, reverse, 0);
   });
+
+  /**
+   * Wait for a condition to return true.
+   *
+   * @param {Function} fn - The function to execute.
+   */
+  client.addCommand('waitUntilInCompass', function(fn) {
+    return progressiveWaitUntil(this.waitUntil.bind(this), fn, 0);
+  });
 }
 
 /**
@@ -255,11 +284,11 @@ function addWaitCommands(client) {
    */
   client.addCommand('waitForIndexCreation', function(name) {
     return this
-      .waitUntil(function() {
+      .waitUntilInCompass(function() {
         return this.getIndexNames().then(function(names) {
           return names.includes(name);
         });
-      }, LONG_TIMEOUT);
+      });
   });
 
   /*
@@ -425,11 +454,11 @@ function addClickCommands(client) {
     return this
       .click(base)
       .waitForVisibleInCompass(base, true)
-      .waitUntil(function() {
+      .waitUntilInCompass(function() {
         return this.getText('div[data-hook=optin-container]').then(function(text) {
           return text.length === 0;
         });
-      }, LONG_TIMEOUT);
+      });
   });
 
   /**
@@ -559,7 +588,10 @@ function addClickCommands(client) {
    * Click the create index button.
    */
   client.addCommand('clickCreateIndexButton', function() {
-    return this.waitForStatusBar().click(selector('open-create-index-modal-button'));
+    const button = selector('open-create-index-modal-button');
+    return this.waitForStatusBar()
+      .waitForVisibleInCompass(button)
+      .click(button);
   });
 
   /**
@@ -1102,19 +1134,32 @@ function addInputCommands(client) {
     let sequence = Promise.resolve();
     if (model.name) {
       sequence = sequence.then(function() {
-        return that.setValue(selector('create-index-modal-name'), model.name)
+        const field = selector('create-index-modal-name');
+        return that
+          .waitForVisibleInCompass(field)
+          .setValue(field, model.name)
       });
     }
     if (model.field) {
       sequence = sequence.then(function() {
         const base = selector('create-index-modal-field-select');
-        return that.click(base).click(`li=${model.field}`);
+        const field = `li=${model.field}`;
+        return that
+          .waitForVisibleInCompass(base)
+          .click(base)
+          .waitForVisibleInCompass(field)
+          .click(field);
       });
     }
     if (model.type) {
       sequence = sequence.then(function() {
         const base = selector('create-index-modal-type-select');
-        return that.click(base).click(`li=${model.type}`);
+        const field = `li=${model.type}`;
+        return that
+          .waitForVisibleInCompass(base)
+          .click(base)
+          .waitForVisibleInCompass(field)
+          .click(field);
       });
     }
     return sequence;