Add e2e tests into CI

.github/workflows/continuous-integration.yml

@@ -19,6 +19,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Setup Python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
@@ -55,3 +60,16 @@ jobs:
           storybookBuildDir: 'webview/storybook-static'
           exitOnceUploaded: true
           autoAcceptChanges: 'main'
+
+      - name: Test E2E
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: yarn test:e2e
+          options: '-screen 0 1600x1200x24'
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: screenshots
+          path: |
+            extension/src/test/e2e/screenshots
+            extension/src/test/e2e/logs

.github/workflows/cross-platform-test.yml

@@ -15,7 +15,7 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
-        os: [macOS-latest, windows-latest]
+        os: [windows-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3

demo/training_metrics.json

@@ -1,5 +1,5 @@
 {
-    "step": 14,
-    "loss": 0.9596208930015564,
-    "acc": 0.7735
-}
+  "step": 14,
+  "loss": 0.9596208930015564,
+  "acc": 0.7735
+}

extension/.gitignore

@@ -1,6 +1,7 @@
 *.vsix
 .vscode-test
 .wdio-vscode-service
+screenshots
 README.md
 LICENSE
 CHANGELOG.md

extension/src/python/index.ts

@@ -1,4 +1,5 @@
 import { join } from 'path'
+import { getVenvBinPath } from './path'
 import { getProcessPlatform } from '../env'
 import { exists } from '../fileSystem'
 import { createProcessWithOutput } from '../processExecution'
@@ -11,11 +12,6 @@ const installPackages = (cwd: string, pythonBin: string, ...args: string[]) => {
   })
 }
 
-export const getVenvBinPath = (cwd: string, envDir: string, name: string) =>
-  getProcessPlatform() === 'win32'
-    ? join(cwd, envDir, 'Scripts', `${name}.exe`)
-    : join(cwd, envDir, 'bin', name)
-
 export const setupVenv = async (
   cwd: string,
   envDir: string,

extension/src/python/path.ts

@@ -0,0 +1,7 @@
+import { join } from 'path'
+import { getProcessPlatform } from '../env'
+
+export const getVenvBinPath = (cwd: string, envDir: string, name: string) =>
+  getProcessPlatform() === 'win32'
+    ? join(cwd, envDir, 'Scripts', `${name}.exe`)
+    : join(cwd, envDir, 'bin', name)

extension/src/test/cli/util.ts

@@ -4,7 +4,7 @@ import { CliExecutor } from '../../cli/executor'
 import { CliReader } from '../../cli/reader'
 import { Config } from '../../config'
 import { exists } from '../../fileSystem'
-import { getVenvBinPath } from '../../python'
+import { getVenvBinPath } from '../../python/path'
 import { dvcDemoPath } from '../util'
 
 const config = {

extension/src/test/e2e/extension.test.ts

@@ -3,22 +3,27 @@ import {
   closeAllEditors,
   dismissAllNotifications,
   getDVCActivityBarIcon,
+  waitForDvcToFinish,
   waitForViewContainerToLoad
 } from './util'
 import { ExperimentsWebview } from './pageObjects/experimentsWebview'
 import { PlotsWebview } from './pageObjects/plotsWebview'
-import { delay } from '../../util/time'
 
 suite('DVC Extension For Visual Studio Code', () => {
-  before('should finish loading the extension', async () => {
+  before('should finish loading the extension', async function () {
+    this.timeout(240000)
     await waitForViewContainerToLoad()
     return dismissAllNotifications()
   })
 
-  // avoid killing any background process after experiments have finished run
-  after(() => delay(30000))
+  after(function () {
+    this.timeout(60000)
+    return waitForDvcToFinish()
+  })
 
-  afterEach(() => browser.switchToFrame(null))
+  afterEach(function () {
+    return browser.switchToFrame(null)
+  })
 
   describe('Activity Bar', () => {
     it('should show the DVC Icon', async () => {
@@ -35,7 +40,7 @@ suite('DVC Extension For Visual Studio Code', () => {
 
       await workbench.executeCommand('DVC: Show Experiments')
 
-      await webview.open()
+      await webview.focus()
 
       await browser.waitUntil(async () => {
         const table = await webview.table$
@@ -44,15 +49,15 @@ suite('DVC Extension For Visual Studio Code', () => {
 
       expect(await webview.table$$).toHaveLength(1)
 
-      await webview.close()
+      await webview.unfocus()
     })
 
     it('should update with a new row for each checkpoint when an experiment is running', async () => {
       const workbench = await browser.getWorkbench()
       const epochs = 15
       await workbench.executeCommand('DVC: Reset and Run Experiment')
 
-      await webview.open()
+      await webview.focus()
 
       await browser.waitUntil(() => webview.expandAllRows())
 
@@ -66,13 +71,19 @@ suite('DVC Extension For Visual Studio Code', () => {
           const currentRows = await webview.row$$
           return currentRows.length >= initialRows.length + epochs
         },
-        { timeout: 120000 }
+        { interval: 5000, timeout: 180000 }
       )
 
+      await webview.unfocus()
+      await waitForDvcToFinish()
+      await webview.focus()
+
       const finalRows = await webview.row$$
 
       expect(finalRows.length).toStrictEqual(initialRows.length + epochs)
-      await webview.close()
+      await webview.unfocus()
+      await waitForDvcToFinish()
+      await workbench.executeCommand('Terminal: Kill All Terminals')
     }).timeout(180000)
   })
 
@@ -87,7 +98,9 @@ suite('DVC Extension For Visual Studio Code', () => {
       const workbench = await browser.getWorkbench()
       await workbench.executeCommand('DVC: Show Plots')
 
-      await webview.open()
+      await waitForDvcToFinish()
+
+      await webview.focus()
 
       await browser.waitUntil(async () => {
         return (await webview.vegaVisualization$$.length) === 6
@@ -101,7 +114,7 @@ suite('DVC Extension For Visual Studio Code', () => {
         expect(plotNotEmpty).toBe(true)
       }
 
-      await webview.close()
+      await webview.unfocus()
     })
   })
 })

extension/src/test/e2e/pageObjects/baseWebview.ts

@@ -17,7 +17,7 @@ export class BaseWebview extends BasePage<
     this.locatorKey = locatorKey
   }
 
-  public async open() {
+  public async focus() {
     const webviewContainer = await this.outerFrame$
 
     await this.outerFrame$.waitForDisplayed()
@@ -31,7 +31,7 @@ export class BaseWebview extends BasePage<
     return browser.switchToFrame(webviewInner)
   }
 
-  public async close() {
+  public async unfocus() {
     await browser.switchToFrame(null)
     await browser.switchToFrame(null)
   }

extension/src/test/e2e/pageObjects/experimentsWebview.ts

@@ -11,7 +11,7 @@ export class ExperimentsWebview extends BaseWebview {
   public async expandAllRows() {
     const expandRowButtons = await this.expandRowButton$$
     for (const button of expandRowButtons) {
-      button.click()
+      await button.click()
     }
     return expandRowButtons.length === 0
   }

extension/src/test/e2e/pageObjects/locators.ts

@@ -7,7 +7,7 @@ export const experiments = {
   ...webview,
   expandRowButton: 'button[title="Expand Row"]',
   row: '[role=row]',
-  table: '[role=table]'
+  table: '[role=tree]'
 }
 
 export const plots = {

extension/src/test/e2e/util.ts

@@ -1,7 +1,11 @@
+import { ChainablePromiseArray, ElementArray } from 'webdriverio'
 import { ViewControl } from 'wdio-vscode-service'
 
-export const dismissAllNotifications = () =>
-  browser.waitUntil(async () => {
+const findProgressBars = (): ChainablePromiseArray<ElementArray> =>
+  $$('.monaco-progress-container')
+
+export const dismissAllNotifications = async (): Promise<void> => {
+  await browser.waitUntil(async () => {
     const workbench = await browser.getWorkbench()
     const notifications = await workbench.getNotifications()
     for (const n of notifications) {
@@ -10,6 +14,23 @@ export const dismissAllNotifications = () =>
     const openNotifications = await workbench.getNotifications()
     return openNotifications.length === 0
   })
+}
+
+const dvcIsWorking = async (): Promise<boolean> => {
+  const workbench = await browser.getWorkbench()
+  const statusBar = workbench.getStatusBar()
+  const statusBarItems = await statusBar.getItems()
+  return statusBarItems.some(
+    statusBarItem =>
+      statusBarItem.includes('loading~spin') && statusBarItem.includes('DVC')
+  )
+}
+
+export const waitForDvcToFinish = async (): Promise<void> => {
+  await browser.waitUntil(async () => !(await dvcIsWorking()), {
+    timeout: 60000
+  })
+}
 
 export const getDVCActivityBarIcon = async (): Promise<ViewControl> => {
   const workbench = await browser.getWorkbench()
@@ -22,8 +43,8 @@ export const getDVCActivityBarIcon = async (): Promise<ViewControl> => {
   return activityBar.getViewControl('DVC') as Promise<ViewControl>
 }
 
-export const waitForViewContainerToLoad = async () => {
-  const initialProgressBars = await $$('.monaco-progress-container')
+export const waitForViewContainerToLoad = async (): Promise<void> => {
+  const initialProgressBars = await findProgressBars()
   await browser.waitUntil(async () => {
     const dvcIcon = await getDVCActivityBarIcon()
     if (!dvcIcon) {
@@ -35,28 +56,42 @@ export const waitForViewContainerToLoad = async () => {
     return !!view
   })
 
-  return browser.waitUntil(async () => {
+  await browser.waitUntil(async () => {
     const numberOfProgressBarsInContainer = 7
-    const currentProgressBars = await $$('.monaco-progress-container')
+    const currentProgressBars = await findProgressBars()
 
-    if (
+    return !(
       currentProgressBars.length <
       initialProgressBars.length + numberOfProgressBarsInContainer
-    ) {
-      return false
-    }
+    )
+  })
+
+  await waitForDvcToFinish()
 
-    for (const progress of currentProgressBars) {
-      if ((await progress.getAttribute('aria-hidden')) !== 'true') {
+  const workbench = await browser.getWorkbench()
+  await workbench.executeCommand('DVC: Pull')
+
+  await browser.waitUntil(
+    async () => {
+      if (await dvcIsWorking()) {
         return false
       }
-    }
 
-    return true
-  })
+      const currentProgressBars = await findProgressBars()
+
+      for (const progress of currentProgressBars) {
+        if (await progress.isDisplayed()) {
+          return false
+        }
+      }
+
+      return true
+    },
+    { timeout: 180000 }
+  )
 }
 
-export const closeAllEditors = async () => {
+export const closeAllEditors = async (): Promise<void> => {
   const workbench = await browser.getWorkbench()
   const editorView = workbench.getEditorView()
   return editorView.closeAllEditors()

extension/src/test/e2e/wdio.conf.ts

@@ -1,13 +1,30 @@
 import { join, resolve } from 'path'
+import { mkdirp } from 'fs-extra'
 import { Options } from '@wdio/types'
+import { getVenvBinPath } from '../../python/path'
+import { Logger } from '../../common/logger'
+
+const screenshotDir = join(__dirname, 'screenshots')
 
 export const config: Options.Testrunner = {
   after: async function () {
     await browser.switchToFrame(null)
     await browser.switchToFrame(null)
   },
+  afterTest: async (test, __, { passed }) => {
+    if (passed) {
+      return
+    }
+
+    Logger.log('Capturing screenshot for debugging')
+
+    await browser.saveScreenshot(
+      join(screenshotDir, `${test.parent} - ${test.title}.png`)
+    )
+  },
   baseUrl: 'http://localhost',
   before: async function () {
+    mkdirp(screenshotDir)
     await browser.setWindowSize(1600, 1200)
   },
   capabilities: [
@@ -17,6 +34,13 @@ export const config: Options.Testrunner = {
       // @ts-expect-error these caps are not typed in WebdriverIO
       'wdio:vscodeOptions': {
         extensionPath: resolve(__dirname, '..', '..', '..'),
+        userSettings: {
+          'dvc.pythonPath': getVenvBinPath(
+            resolve(__dirname, '..', '..', '..', '..', 'demo'),
+            '.env',
+            'python'
+          )
+        },
         verboseLogging: false,
         workspacePath: resolve(__dirname, '..', '..', '..', '..', 'demo')
       }