Spec level retry for failing integration tests (#312)

kartikgupta-db · web-flow · commit da00229c2e80 · 2022-12-20T13:02:13.000+01:00
diff --git a/packages/databricks-vscode/src/test/e2e/configure.e2e.ts b/packages/databricks-vscode/src/test/e2e/configure.e2e.ts
@@ -41,6 +41,10 @@ describe("Configure Databricks Extension", async function () {
         host = process.env.DATABRICKS_HOST;
 
         workbench = await browser.getWorkbench();
+    });
+
+    it("should install vscode python extension", async function () {
+        this.retries(1);
         await waitForPythonExtension();
     });
 
diff --git a/packages/databricks-vscode/src/test/e2e/run_job_on_cluster.e2e.ts b/packages/databricks-vscode/src/test/e2e/run_job_on_cluster.e2e.ts
@@ -3,13 +3,14 @@ import * as fs from "fs/promises";
 import assert from "node:assert";
 import {
     getViewSection,
-    getViewSubSection,
+    startSyncIfStopped,
     waitForPythonExtension,
+    waitForSyncComplete,
     waitForTreeItems,
 } from "./utils";
-import {sleep, TreeItem} from "wdio-vscode-service";
+import {sleep} from "wdio-vscode-service";
 
-describe("Run python on cluster", async function () {
+describe("Run job on cluster", async function () {
     let projectDir: string;
     this.timeout(2 * 60 * 1000);
 
@@ -45,59 +46,17 @@ describe("Run python on cluster", async function () {
         const section = await getViewSection("CONFIGURATION");
         assert(section);
         await waitForTreeItems(section);
-        await waitForPythonExtension();
     });
 
-    beforeEach(async () => {
-        const section = await getViewSection("CLUSTERS");
-        await section?.collapse();
-
-        const repoConfigItem = await getViewSubSection("CONFIGURATION", "Repo");
-        assert(repoConfigItem);
-
-        let status: TreeItem | undefined = undefined;
-        for (const i of await repoConfigItem.getChildren()) {
-            if ((await i.getLabel()).includes("State:")) {
-                status = i;
-                break;
-            }
-        }
-        assert(status);
-        if ((await status.getDescription())?.includes("STOPPED")) {
-            const buttons = await repoConfigItem.getActionButtons();
-            await buttons[0].elem.click();
-        }
-
-        await browser.waitUntil(
-            async () => {
-                const repoConfigItem = await getViewSubSection(
-                    "CONFIGURATION",
-                    "Repo"
-                );
-                assert(repoConfigItem);
-
-                status = undefined;
-                for (const i of await repoConfigItem.getChildren()) {
-                    if ((await i.getLabel()).includes("State:")) {
-                        status = i;
-                        break;
-                    }
-                }
-                assert(status);
-                const description = await status?.getDescription();
-                return (
-                    description !== undefined &&
-                    description.includes("WATCHING_FOR_CHANGES")
-                );
-            },
-            {
-                timeout: 20000,
-                timeoutMsg: "Couldn't finish sync in 20s",
-            }
-        );
+    it("should install vscode python extension", async function () {
+        this.retries(1);
+        await waitForPythonExtension();
     });
 
     it("should run a python notebook as a job on a cluster", async () => {
+        await startSyncIfStopped();
+        await waitForSyncComplete();
+
         const workbench = await driver.getWorkbench();
         const editorView = workbench.getEditorView();
         await editorView.closeAllEditors();
@@ -161,6 +120,9 @@ describe("Run python on cluster", async function () {
     });
 
     it("should run a python file as a job on a cluster", async () => {
+        await startSyncIfStopped();
+        await waitForSyncComplete();
+
         const workbench = await driver.getWorkbench();
         const editorView = workbench.getEditorView();
         await editorView.closeAllEditors();
diff --git a/packages/databricks-vscode/src/test/e2e/run_on_cluster.e2e.ts b/packages/databricks-vscode/src/test/e2e/run_on_cluster.e2e.ts
@@ -3,8 +3,9 @@ import * as fs from "fs/promises";
 import assert from "node:assert";
 import {
     getViewSection,
-    getViewSubSection,
+    startSyncIfStopped,
     waitForPythonExtension,
+    waitForSyncComplete,
     waitForTreeItems,
 } from "./utils";
 import {sleep} from "wdio-vscode-service";
@@ -36,6 +37,10 @@ describe("Run python on cluster", async function () {
             path.join(projectDir, "hello.py"),
             `spark.sql('SELECT "hello world"').show()`
         );
+    });
+
+    it("should install vscode python extension", async function () {
+        this.retries(1);
         await waitForPythonExtension();
     });
 
@@ -46,25 +51,8 @@ describe("Run python on cluster", async function () {
     });
 
     it("should start syncing", async () => {
-        const section = await getViewSection("CLUSTERS");
-        await section?.collapse();
-
-        const repoConfigItem = await getViewSubSection("CONFIGURATION", "Repo");
-        assert(repoConfigItem);
-        const buttons = await repoConfigItem.getActionButtons();
-        await buttons[0].elem.click();
-
-        // wait for sync to finish
-        const workbench = await driver.getWorkbench();
-        const terminalView = await workbench.getBottomBar().openTerminalView();
-
-        while (true) {
-            await sleep(500);
-            const text = await terminalView.getText();
-            if (text.includes("Sync Complete")) {
-                break;
-            }
-        }
+        await startSyncIfStopped();
+        await waitForSyncComplete();
     });
 
     it("should run a python file on a cluster", async () => {
diff --git a/packages/databricks-vscode/src/test/e2e/utils.ts b/packages/databricks-vscode/src/test/e2e/utils.ts
@@ -3,36 +3,45 @@ import {
     CustomTreeSection,
     sleep,
     TreeItem,
+    ViewControl,
     ViewSection,
 } from "wdio-vscode-service";
 
-export type ViewSectionType = "CLUSTERS" | "CONFIGURATION";
-export async function getViewSection(
-    name: ViewSectionType
-): Promise<ViewSection | undefined> {
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const ViewSectionTypes = ["CLUSTERS", "CONFIGURATION"] as const;
+export type ViewSectionType = typeof ViewSectionTypes[number];
+
+export async function findViewSection(name: ViewSectionType) {
     const workbench = await browser.getWorkbench();
 
-    let control;
-    for (let i = 0; i <= 10; i++) {
-        if (i === 10) {
-            assert.fail(`Can't find view control "${name}"`);
-        }
-        control = await workbench.getActivityBar().getViewControl("Databricks");
-        if (control) {
-            break;
+    let control: ViewControl | undefined;
+    await browser.waitUntil(
+        async () => {
+            control = await workbench
+                .getActivityBar()
+                .getViewControl("Databricks");
+            if (!control) {
+                return false;
+            }
+            return true;
+        },
+        {
+            timeout: 10 * 1000,
+            interval: 1 * 1000,
+            timeoutMsg: `Can't find view control "${name}"`,
         }
-        await sleep(500);
-    }
-    assert.ok(control);
-
-    const view = await control.openView();
-    assert.ok(view);
-
-    const content = await view.getContent();
-    assert.ok(content);
+    );
+    const view = await (await control?.openView())
+        ?.getContent()
+        ?.getSection(name);
+    return view;
+}
 
-    const section = await content.getSection(name);
-    assert.ok(section);
+export async function getViewSection(
+    name: ViewSectionType
+): Promise<ViewSection | undefined> {
+    const section = await findViewSection(name);
+    assert(section);
 
     await section.expand();
     await (await section.elem).click();
@@ -43,6 +52,11 @@ export async function getViewSubSection(
     section: ViewSectionType,
     subSection: string
 ): Promise<TreeItem | undefined> {
+    for (const s of ViewSectionTypes) {
+        if (s !== section) {
+            await (await findViewSection(s))?.collapse();
+        }
+    }
     const sectionView = await getViewSection(section);
 
     if (!sectionView) {
@@ -87,18 +101,30 @@ export async function waitForPythonExtension() {
     assert(section);
     const welcome = await section.findWelcomeContent();
     assert(welcome);
-    sleep(1000);
+    sleep(5000);
+
     const workbench = await browser.getWorkbench();
-    const notifs = await workbench.getNotifications();
-    for (const n of notifs) {
-        if (
-            (await n.getActions()).find(
-                (btn) => btn.getTitle() === "Install and Reload"
-            ) !== undefined
-        ) {
-            await n.takeAction("Install and Reload");
+    browser.waitUntil(
+        async () => {
+            const notifs = await workbench.getNotifications();
+            let found = false;
+            for (const n of notifs) {
+                if (
+                    (await n.getActions()).find(
+                        (btn) => btn.getTitle() === "Install and Reload"
+                    ) !== undefined
+                ) {
+                    await n.takeAction("Install and Reload");
+                    found = true;
+                }
+            }
+
+            return found;
+        },
+        {
+            timeout: 10 * 1000,
         }
-    }
+    );
 
     await browser.waitUntil(
         async () =>
@@ -108,16 +134,92 @@ export async function waitForPythonExtension() {
                 )?.getTitle()
             )?.includes("README.quickstart.md") === true,
         {
-            timeout: 120000,
+            timeout: 1.5 * 60 * 1000,
             timeoutMsg:
                 "Timeout when installing python extension and reloading",
         }
     );
 
-    sleep(500);
+    sleep(1000);
+    const notifs = await workbench.getNotifications();
     try {
         for (const n of notifs) {
             await n.dismiss();
         }
     } catch {}
 }
+
+export async function waitForSyncComplete() {
+    await browser.waitUntil(
+        async () => {
+            const repoConfigItem = await getViewSubSection(
+                "CONFIGURATION",
+                "Repo"
+            );
+            if (repoConfigItem === undefined) {
+                return false;
+            }
+            await repoConfigItem.expand();
+
+            let status: TreeItem | undefined = undefined;
+            for (const i of await repoConfigItem.getChildren()) {
+                if ((await i.getLabel()).includes("State:")) {
+                    status = i;
+                    break;
+                }
+            }
+            if (status === undefined) {
+                return false;
+            }
+
+            const description = await status?.getDescription();
+            return (
+                description !== undefined &&
+                description.includes("WATCHING_FOR_CHANGES")
+            );
+        },
+        {
+            timeout: 60000,
+            interval: 2000,
+            timeoutMsg: "Couldn't finish sync in 1m",
+        }
+    );
+}
+
+export async function startSyncIfStopped() {
+    browser.waitUntil(
+        async () => {
+            const repoConfigItem = await getViewSubSection(
+                "CONFIGURATION",
+                "Repo"
+            );
+            if (repoConfigItem === undefined) {
+                return false;
+            }
+            await repoConfigItem.expand();
+
+            let status: TreeItem | undefined = undefined;
+            for (const i of await repoConfigItem.getChildren()) {
+                if ((await i.getLabel()).includes("State:")) {
+                    status = i;
+                    break;
+                }
+            }
+            if (status === undefined) {
+                return false;
+            }
+
+            if ((await status.getDescription())?.includes("STOPPED")) {
+                const buttons = await repoConfigItem.getActionButtons();
+                if (buttons.length === 0) {
+                    return false;
+                }
+                await buttons[0].elem.click();
+            }
+            return true;
+        },
+        {
+            timeout: 20000,
+        }
+    );
+}
diff --git a/packages/databricks-vscode/src/test/e2e/wdio.conf.ts b/packages/databricks-vscode/src/test/e2e/wdio.conf.ts
