Improve test reliability (#2777)

* Use driver.wait for condition

* Use global timeout values

* Wait until button is hid

* Wait for button visibility before taking screenshot

* Update minNumActivitiesShown

* Wait for scroll to bottom button visibility

* Typo

* Wait until language change

* Do 5 consecutive checks

* Add entry

* Fix tests

* Use another timeout for images

* Update entry

* Update __tests__/setup/conditions/scrollToBottomCompleted.js

Co-Authored-By: Corina <14900841+corinagum@users.noreply.github.com>

* Rename to negationOf

Co-authored-by: Corina <14900841+corinagum@users.noreply.github.com>

CHANGELOG.md

@@ -32,6 +32,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
    -  [`abort-controller`](https://npmjs.com/package/abort-controller)
    -  [`event-target-shim`](https://npmjs.com/package/event-target-shim)
    -  [`p-defer`](https://npmjs.com/package/p-defer)
+-  Fixes the following issues and improves test reliability, by [@compulim](https://github.com/compulim) in PR [#XXX](https://github.com/microsoft/BotFramework-WebChat/pull/XXX)
+   - Fixes [#2612](https://github.com/microsoft/BotFramework-WebChat/issues/2612). Wait until language change
+   - Fixes [#2653](https://github.com/microsoft/BotFramework-WebChat/issues/2653). Scroll-to-bottom check will do 5 consecutive checks to determine stickiness.
+   - Fixes [#2691](https://github.com/microsoft/BotFramework-WebChat/issues/2691). Wait until button is shown/hid before taking screenshot
+   - Fixes [#2737](https://github.com/microsoft/BotFramework-WebChat/issues/2737). Use `driver.wait` for conditions
+   - Fixes [#2776](https://github.com/microsoft/BotFramework-WebChat/issues/2776). Wait until button is shown/hid before taking screenshot
+   - Use a new timeout `fetchImage` for images
 
 ### Changed
 

__tests__/adaptiveCards.js

@@ -20,7 +20,7 @@ test('breakfast card', async () => {
   await pageObjects.sendMessageViaSendBox('card breakfast', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -40,7 +40,7 @@ test('breakfast card with custom host config', async () => {
   await pageObjects.sendMessageViaSendBox('card breakfast', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -60,7 +60,7 @@ test('breakfast card with custom style options', async () => {
   await pageObjects.sendMessageViaSendBox('card breakfast', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -74,7 +74,7 @@ test('disable card inputs', async () => {
   await pageObjects.sendMessageViaSendBox('card inputs', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
   await driver.wait(scrollToBottomCompleted(), timeouts.scrollToBottom);
 
   await driver.executeScript(() => {

__tests__/basic.js

@@ -16,7 +16,7 @@ test('setup', async () => {
   await pageObjects.sendMessageViaSendBox('layout carousel', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -33,7 +33,7 @@ test('long URLs with break-word', async () => {
   );
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -52,7 +52,7 @@ test('long URLs with break-all', async () => {
   );
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -69,7 +69,7 @@ test('long URLs with keep-all', async () => {
   });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -96,7 +96,7 @@ test('hero card with a long title and richCardWrapTitle set to true', async () =
   await pageObjects.sendMessageViaSendBox('herocard long title', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -110,7 +110,7 @@ test('hero card with a long title and richCardWrapTitle set to default value', a
   await pageObjects.sendMessageViaSendBox('herocard long title', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -124,7 +124,7 @@ test('thumbnail card with a long title and richCardWrapTitle set to true', async
   await pageObjects.sendMessageViaSendBox('thumbnailcard long title', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 
@@ -138,7 +138,7 @@ test('thumbnail card with a long title and richCardWrapTitle set to default valu
   await pageObjects.sendMessageViaSendBox('thumbnailcard long title', { waitForSend: true });
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), 2000);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 

__tests__/bubbleBorder.js

@@ -1,6 +1,5 @@
 import { imageSnapshotOptions, timeouts } from './constants.json';
 
-import allImagesLoaded from './setup/conditions/allImagesLoaded';
 import minNumActivitiesShown from './setup/conditions/minNumActivitiesShown';
 import uiConnected from './setup/conditions/uiConnected';
 

__tests__/bubbleNub.js

@@ -14,7 +14,7 @@ async function sendMessageAndMatchSnapshot(driver, pageObjects, message) {
   await pageObjects.sendMessageViaSendBox(message);
 
   await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-  await driver.wait(allImagesLoaded(), timeouts.fetch);
+  await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
   const base64PNG = await driver.takeScreenshot();
 

__tests__/cardActionMiddleware.js

@@ -10,6 +10,8 @@ import uiConnected from './setup/conditions/uiConnected';
 // selenium-webdriver API doc:
 // https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html
 
+jest.setTimeout(timeouts.test);
+
 test('card action "openUrl"', async () => {
   const { driver, pageObjects } = await setupWebDriver({
     props: {
@@ -42,7 +44,7 @@ test('card action "openUrl"', async () => {
   const base64PNG = await driver.takeScreenshot();
 
   expect(base64PNG).toMatchImageSnapshot(imageSnapshotOptions);
-}, 60000);
+});
 
 test('card action "signin"', async () => {
   const { driver, pageObjects } = await setupWebDriver({
@@ -85,4 +87,4 @@ test('card action "signin"', async () => {
   const base64PNG = await driver.takeScreenshot();
 
   expect(base64PNG).toMatchImageSnapshot(imageSnapshotOptions);
-}, 60000);
+});

__tests__/carousel.js

@@ -19,7 +19,7 @@ describe('carousel without avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
@@ -43,7 +43,7 @@ describe('carousel without avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
@@ -67,7 +67,7 @@ describe('carousel without avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout double', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });
@@ -79,7 +79,7 @@ describe('carousel without avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout double', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });
@@ -91,7 +91,7 @@ describe('carousel without avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout single carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });
@@ -103,7 +103,7 @@ describe('carousel without avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout single carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });
@@ -119,7 +119,7 @@ describe('carousel with avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
@@ -143,7 +143,7 @@ describe('carousel with avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
 
@@ -167,7 +167,7 @@ describe('carousel with avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout double', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });
@@ -179,7 +179,7 @@ describe('carousel with avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout double', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });
@@ -191,7 +191,7 @@ describe('carousel with avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout single carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });
@@ -203,7 +203,7 @@ describe('carousel with avatar initials', () => {
     await pageObjects.sendMessageViaSendBox('layout single carousel', { waitForSend: true });
 
     await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });

__tests__/clockSkew.js

@@ -150,7 +150,7 @@ describe('Clock skew', () => {
     await driver.executeScript(() => window.WebChatTest.releaseActivity(3));
     await driver.wait(minNumActivitiesShown(5), timeouts.directLine);
     await driver.wait(allOutgoingActivitiesSent(), timeouts.directLine);
-    await driver.wait(allImagesLoaded(), timeouts.fetch);
+    await driver.wait(allImagesLoaded(), timeouts.fetchImage);
 
     expect(await driver.takeScreenshot()).toMatchImageSnapshot(imageSnapshotOptions);
   });

__tests__/constants.json

@@ -7,6 +7,7 @@
   "timeouts": {
     "directLine": 15000,
     "fetch": 2500,
+    "fetchImage": 5000,
     "navigation": 10000,
     "postActivity": 30000,
     "scrollToBottom": 2000,

__tests__/focus.js

@@ -3,6 +3,7 @@ import { By, Key } from 'selenium-webdriver';
 import { imageSnapshotOptions, timeouts } from './constants.json';
 import allOutgoingActivitiesSent from './setup/conditions/allOutgoingActivitiesSent';
 import minNumActivitiesShown from './setup/conditions/minNumActivitiesShown';
+import negationOf from './setup/conditions/negationOf.js';
 import scrollToBottomCompleted from './setup/conditions/scrollToBottomCompleted';
 import sendBoxTextBoxFocused from './setup/conditions/sendBoxTextBoxFocused';
 import suggestedActionsShown from './setup/conditions/suggestedActionsShown';
@@ -22,7 +23,7 @@ test('should not focus send box after clicking on send button', async () => {
   await pageObjects.typeOnSendBox('echo 123');
   await pageObjects.clickSendButton();
 
-  await expect(sendBoxTextBoxFocused().fn(driver)).resolves.toBeFalsy();
+  await driver.wait(negationOf(sendBoxTextBoxFocused()), timeouts.ui);
 });
 
 // Verification of fix of #1971, https://github.com/microsoft/BotFramework-WebChat/issues/1971
@@ -36,7 +37,7 @@ test('SHOULD focus send box after clicking on suggested actions', async () => {
 
   await pageObjects.clickSuggestedActionButton(0);
 
-  await expect(sendBoxTextBoxFocused().fn(driver)).resolves.toBeTruthy();
+  await driver.wait(sendBoxTextBoxFocused(), timeouts.ui);
 });
 
 // Verification of fix of #1971, https://github.com/microsoft/BotFramework-WebChat/issues/1971
@@ -47,7 +48,7 @@ test('should focus send box after pressing ENTER to send message', async () => {
 
   await pageObjects.typeOnSendBox('echo 123', Key.RETURN);
 
-  await expect(sendBoxTextBoxFocused().fn(driver)).resolves.toBeTruthy();
+  await driver.wait(sendBoxTextBoxFocused(), timeouts.ui);
 });
 
 describe('type focus sink', () => {
@@ -60,7 +61,7 @@ describe('type focus sink', () => {
 
     await transcript.click();
 
-    await expect(sendBoxTextBoxFocused().fn(driver)).resolves.toBeFalsy();
+    await driver.wait(negationOf(sendBoxTextBoxFocused()), timeouts.ui);
 
     await driver
       .actions()
@@ -111,14 +112,14 @@ describe('type focus sink', () => {
 
     await transcript.click();
 
-    await expect(sendBoxTextBoxFocused().fn(driver)).resolves.toBeFalsy();
+    await driver.wait(negationOf(sendBoxTextBoxFocused()), timeouts.ui);
 
     await driver
       .actions()
       .sendKeys(Key.SHIFT)
       .perform();
 
-    await expect(sendBoxTextBoxFocused().fn(driver)).resolves.toBeFalsy();
+    await driver.wait(negationOf(sendBoxTextBoxFocused()), timeouts.ui);
   });
 
   test('should paste into the send box when focus is on the transcript', async () => {
@@ -132,7 +133,7 @@ describe('type focus sink', () => {
 
     await transcript.click();
 
-    await expect(sendBoxTextBoxFocused().fn(driver)).resolves.toBeFalsy();
+    await driver.wait(negationOf(sendBoxTextBoxFocused()), timeouts.ui);
 
     await driver
       .actions()

__tests__/hooks/useLanguage.js

@@ -12,17 +12,17 @@ test('getter should return language set in props', async () => {
     }
   });
 
-  const [groupTimestamp] = await pageObjects.runHook('useLanguage');
+  const [language] = await pageObjects.runHook('useLanguage');
 
-  expect(groupTimestamp).toMatchInlineSnapshot(`"zh-YUE"`);
+  expect(language).toMatchInlineSnapshot(`"zh-YUE"`);
 });
 
 test('getter should return default language if not set in props', async () => {
   const { pageObjects } = await setupWebDriver();
 
-  const [groupTimestamp] = await pageObjects.runHook('useLanguage');
+  const [language] = await pageObjects.runHook('useLanguage');
 
-  expect(groupTimestamp).toMatchInlineSnapshot(`"en-US"`);
+  expect(language).toMatchInlineSnapshot(`"en-US"`);
 });
 
 test('setter should be undefined', async () => {

__tests__/hooks/useSendBoxSpeechInterimsVisible.js

@@ -1,6 +1,6 @@
 import { timeouts } from '../constants.json';
 
-import negate from '../setup/conditions/negate';
+import negationOf from '../setup/conditions/negationOf';
 import speechRecognitionStartCalled from '../setup/conditions/speechRecognitionStartCalled';
 import speechSynthesisUtterancePended from '../setup/conditions/speechSynthesisUtterancePended';
 import uiConnected from '../setup/conditions/uiConnected';
@@ -73,7 +73,7 @@ test('sendBoxSpeechInterimsVisible should return false when synthesizing', async
 
   await pageObjects.clickMicrophoneButton();
 
-  await driver.wait(negate(speechSynthesisUtterancePended()), timeouts.ui);
+  await driver.wait(negationOf(speechSynthesisUtterancePended()), timeouts.ui);
 
   await expect(
     pageObjects.runHook(