More test reliability (#1717)

* Use jest.setTimeout instead

* Wait until postback response

* Run test suite one at a time

* Whitelist logs instead of blacklist

* Make caret transparent instead of blurring the focused element

* Typo

* Remove cursor

* Disable Docker log

* Disable Web Socket

* Correct retry count

* On retry, restart the browser

* Use concurrency of 2 on Travis CI

* Wait on driver.quit

* Use Direct Line timeout

* Re-enable Web Socket

* Revert concurrency after no improvements

.travis.yml

@@ -25,8 +25,11 @@ install:
 script:
 - docker-compose up --build -d
 - docker-compose exec webchat ls -la
-- npm test -- --coverageReporters=lcov --coverageReporters=text
-- docker-compose logs
+- npm test -- --coverageReporters=lcov --coverageReporters=text --maxWorkers=1
+
+# Enable Docker log if investigation is needed
+# - docker-compose logs
+
 - docker-compose down --rmi all
 - npm run coveralls
 

__tests__/basic.js

@@ -6,6 +6,8 @@ import minNumActivitiesShown from './setup/conditions/minNumActivitiesShown';
 // selenium-webdriver API doc:
 // https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html
 
+jest.setTimeout(timeouts.test);
+
 test('setup', async () => {
   const { driver, pageObjects } = await setupWebDriver();
 
@@ -14,10 +16,7 @@ test('setup', async () => {
   await driver.wait(minNumActivitiesShown(3), 2000);
   await driver.wait(allImagesLoaded(), 2000);
 
-  // Hide cursor before taking screenshot
-  await pageObjects.hideCursor();
-
   const base64PNG = await driver.takeScreenshot();
 
   expect(base64PNG).toMatchImageSnapshot(imageSnapshotOptions);
-}, timeouts.test);
+});

__tests__/carousel.js

@@ -8,6 +8,8 @@ import minNumActivitiesShown from './setup/conditions/minNumActivitiesShown';
 // selenium-webdriver API doc:
 // https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html
 
+jest.setTimeout(timeouts.test);
+
 describe('carousel without avatar initials', () => {
   test('4 attachments and no message', async () => {
     const { driver, pageObjects } = await setupWebDriver();
@@ -17,9 +19,6 @@ describe('carousel without avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
     const rightFlipper = await driver.findElement(By.css('button[aria-label="Right"]'));
@@ -33,7 +32,7 @@ describe('carousel without avatar initials', () => {
     await driver.sleep(1000);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('4 attachments and message', async () => {
     const { driver, pageObjects } = await setupWebDriver();
@@ -43,9 +42,6 @@ describe('carousel without avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
     const rightFlipper = await driver.findElement(By.css('button[aria-label="Right"]'));
@@ -59,7 +55,7 @@ describe('carousel without avatar initials', () => {
     await driver.sleep(1000);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('2 attachments', async () => {
     const { driver, pageObjects } = await setupWebDriver();
@@ -69,11 +65,8 @@ describe('carousel without avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('2 attachments with wide screen', async () => {
     const { driver, pageObjects } = await setupWebDriver({ width: 640 });
@@ -83,11 +76,8 @@ describe('carousel without avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('1 attachment', async () => {
     const { driver, pageObjects } = await setupWebDriver();
@@ -97,11 +87,8 @@ describe('carousel without avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('1 attachment with wide screen', async () => {
     const { driver, pageObjects } = await setupWebDriver({ width: 640 });
@@ -111,11 +98,8 @@ describe('carousel without avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 });
 
 describe('carousel with avatar initials', () => {
@@ -129,9 +113,6 @@ describe('carousel with avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
     const rightFlipper = await driver.findElement(By.css('button[aria-label="Right"]'));
@@ -145,7 +126,7 @@ describe('carousel with avatar initials', () => {
     await driver.sleep(1000);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('4 attachments and message', async () => {
     const { driver, pageObjects } = await setupWebDriver({ props: WEB_CHAT_PROPS });
@@ -155,9 +136,6 @@ describe('carousel with avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
     const rightFlipper = await driver.findElement(By.css('button[aria-label="Right"]'));
@@ -171,7 +149,7 @@ describe('carousel with avatar initials', () => {
     await driver.sleep(1000);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('2 attachments', async () => {
     const { driver, pageObjects } = await setupWebDriver({ props: WEB_CHAT_PROPS });
@@ -181,11 +159,8 @@ describe('carousel with avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('2 attachments with wide screen', async () => {
     const { driver, pageObjects } = await setupWebDriver({ props: WEB_CHAT_PROPS, width: 640 });
@@ -195,11 +170,8 @@ describe('carousel with avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('1 attachment', async () => {
     const { driver, pageObjects } = await setupWebDriver({ props: WEB_CHAT_PROPS });
@@ -209,11 +181,8 @@ describe('carousel with avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('1 attachment with wide screen', async () => {
     const { driver, pageObjects } = await setupWebDriver({ props: WEB_CHAT_PROPS, width: 640 });
@@ -223,9 +192,6 @@ describe('carousel with avatar initials', () => {
     await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
     await driver.wait(allImagesLoaded(), timeouts.fetch);
 
-    // Hide cursor before taking screenshot
-    await pageObjects.hideCursor();
-
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 });

__tests__/offlineUI.js

@@ -5,6 +5,8 @@ import { imageSnapshotOptions, timeouts } from './constants.json';
 // selenium-webdriver API doc:
 // https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html
 
+jest.setTimeout(timeouts.test);
+
 const allOutgoingMessagesFailed = new Condition('All outgoing messages to fail sending', driver => {
   return driver.executeScript(() => {
     const { store } = window.WebChatTest;
@@ -53,7 +55,7 @@ describe('offline UI', async () => {
     const base64PNG = await driver.takeScreenshot();
 
     expect(base64PNG).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('should show "unable to connect" UI when credentials are incorrect', async () => {
     const { driver } = await setupWebDriver({
@@ -80,7 +82,7 @@ describe('offline UI', async () => {
     const base64PNG = await driver.takeScreenshot();
 
     expect(base64PNG).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('should display "Send failed. Retry" when activity is not able to send', async () => {
     const { driver, pageObjects } = await setupWebDriver({
@@ -117,7 +119,7 @@ describe('offline UI', async () => {
     const base64PNG = await driver.takeScreenshot();
 
     expect(base64PNG).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 
   test('should display "Send failed. Retry" when activity is sent but not acknowledged', async() => {
     const { driver, pageObjects } = await setupWebDriver({
@@ -170,5 +172,5 @@ describe('offline UI', async () => {
     const base64PNG = await driver.takeScreenshot();
 
     expect(base64PNG).toMatchImageSnapshot(imageSnapshotOptions);
-  }, timeouts.test);
+  });
 });

__tests__/sendTypingIndicator.js

@@ -6,33 +6,35 @@ import minNumActivitiesShown from './setup/conditions/minNumActivitiesShown';
 // selenium-webdriver API doc:
 // https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html
 
+jest.setTimeout(timeouts.test);
+
 test('Send typing indicator', async () => {
   const { driver, pageObjects } = await setupWebDriver({ props: { sendTypingIndicator: true } });
 
   await pageObjects.sendMessageViaSendBox('echo-typing');
 
-  await driver.wait(minNumActivitiesShown(3), 2000);
+  await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
 
   const input = await driver.findElement(By.css('input[type="text"]'));
 
   await input.sendKeys('ABC');
 
   // Typing indicator takes longer to come back
   await driver.wait(minNumActivitiesShown(4), 5000);
-}, timeouts.test);
+});
 
 // TODO: [P3] Take this deprecation code out when releasing on or after January 13 2020
 test('Send typing indicator using deprecated props', async () => {
   const { driver, pageObjects } = await setupWebDriver({ props: { sendTyping: true } });
 
   await pageObjects.sendMessageViaSendBox('echo-typing');
 
-  await driver.wait(minNumActivitiesShown(3), 2000);
+  await driver.wait(minNumActivitiesShown(3), timeouts.directLine);
 
   const input = await driver.findElement(By.css('input[type="text"]'));
 
   await input.sendKeys('ABC');
 
   // Typing indicator takes longer to come back
   await driver.wait(minNumActivitiesShown(4), 5000);
-}, timeouts.test);
+});

__tests__/setup/conditions/allOutgoingActivitiesSent.js

@@ -1,7 +1,7 @@
 import { Condition } from 'selenium-webdriver';
 
 export default function () {
-  return new Condition('Waiting for Direct Line to connect', async driver => {
+  return new Condition('all outgoing activities to be sent', async driver => {
     return await driver.executeScript(() => {
       const { store } = window.WebChatTest;
       const { activities } = store.getState();

__tests__/setup/pageObjects/index.js

@@ -1,5 +1,4 @@
 import dispatchAction from './dispatchAction';
-import hideCursor from './hideCursor';
 import pingBot from './pingBot';
 import sendMessageViaSendBox from './sendMessageViaSendBox';
 
@@ -14,7 +13,6 @@ function mapMap(map, mapper) {
 export default function (driver) {
   return mapMap({
     dispatchAction,
-    hideCursor,
     pingBot,
     sendMessageViaSendBox
   }, fn => fn.bind(null, driver));

__tests__/setup/retry.js

@@ -1,7 +1,7 @@
 export default async function (fn, retries) {
   let lastErr;
 
-  for (; retries > 0; retries--) {
+  for (; retries >= 0; retries--) {
     try {
       return await fn();
     } catch (err) {

__tests__/setup/setupTestFramework.js

@@ -34,12 +34,11 @@ global.setupWebDriver = async options => {
   options = { ...DEFAULT_OPTIONS, ...options };
 
   if (!driverPromise) {
-    driverPromise = (async () => {
+    driverPromise = retry(async () => {
       let { baseURL, builder } = await setupTestEnvironment(BROWSER_NAME, new Builder(), options);
       const driver = builder.build();
-      const pageObjects = createPageObjects(driver);
 
-      return await retry(async () => {
+      try {
         // If the baseURL contains $PORT, it means it requires us to fill-in
         if (/\$PORT/i.test(baseURL)) {
           const { port } = await global.setupWebServer();
@@ -72,11 +71,17 @@ global.setupWebDriver = async options => {
 
         await driver.wait(webChatLoaded(), timeouts.navigation);
 
+        const pageObjects = createPageObjects(driver);
+
         options.pingBotOnLoad && await pageObjects.pingBot();
 
         return { driver, pageObjects };
-      }, 3);
-    })();
+      } catch (err) {
+        await driver.quit();
+
+        throw err;
+      }
+    }, 3);
   }
 
   return await driverPromise;
@@ -123,7 +128,7 @@ afterEach(async () => {
       global.__coverage__ = await driver.executeScript(() => window.__coverage__);
 
       ((await driver.executeScript(() => window.__console__)) || [])
-        .filter(([type]) => type !== 'info' && type !== 'log')
+        .filter(([type]) => type === 'error' && type === 'warn')
         .forEach(([type, message]) => {
           console.log(`${ type }: ${ message }`);
         });