Add e2e tests (wdio-vscode-service)

extension/.gitignore

@@ -1,5 +1,6 @@
 *.vsix
 .vscode-test
+.wdio-vscode-service
 README.md
 LICENSE
 CHANGELOG.md

extension/package.json

@@ -1493,8 +1493,9 @@
     "dev": "shx rm -rf dist && tsc -watch -p ./",
     "build": "yarn webpack --mode production",
     "package": "vsce package --yarn -o ./dvc.vsix",
-    "test-vscode-build": "tsc -p .",
-    "test-vscode-run": "node ./dist/test/runTest.js",
+    "test-build": "tsc -p .",
+    "test-vscode": "node ./dist/test/runTest.js",
+    "test-e2e": "wdio run ./src/test/e2e/wdio.conf.ts",
     "test": "jest --collect-coverage",
     "cover-vscode-run": "node ./scripts/coverIntegrationTests.js",
     "vscode:prepublish": ""
@@ -1536,6 +1537,10 @@
     "@types/uuid": "8.3.4",
     "@types/vscode": "1.64.0",
     "@vscode/test-electron": "2.1.5",
+    "@wdio/cli": "^7.20.5",
+    "@wdio/local-runner": "^7.20.5",
+    "@wdio/mocha-framework": "^7.20.3",
+    "@wdio/spec-reporter": "^7.20.3",
     "chai": "4.3.6",
     "chai-as-promised": "7.1.1",
     "clean-webpack-plugin": "4.0.0",
@@ -1553,6 +1558,8 @@
     "ts-loader": "9.3.1",
     "vsce": "2.9.2",
     "vscode-uri": "3.0.3",
+    "wdio-vscode-service": "^3.0.2",
+    "webdriverio": "^7.20.5",
     "webpack": "5.73.0",
     "webpack-cli": "4.10.0"
   },

extension/src/test/e2e/extension.test.ts

@@ -0,0 +1,107 @@
+import { suite, before, describe, it } from 'mocha'
+import {
+  closeAllEditors,
+  dismissAllNotifications,
+  getDVCActivityBarIcon,
+  waitForViewContainerToLoad
+} from './util'
+import { ExperimentsWebview } from './pageObjects/experimentsWebview'
+import { PlotsWebview } from './pageObjects/plotsWebview'
+import { delay } from '../../util/time'
+
+suite('DVC Extension For Visual Studio Code', () => {
+  before('should finish loading the extension', async () => {
+    await waitForViewContainerToLoad()
+    return dismissAllNotifications()
+  })
+
+  // avoid killing any background process after experiments have finished run
+  after(() => delay(30000))
+
+  afterEach(() => browser.switchToFrame(null))
+
+  describe('Activity Bar', () => {
+    it('should show the DVC Icon', async () => {
+      const dvcIcon = await getDVCActivityBarIcon()
+      expect(await dvcIcon.getTitle()).toBe('DVC')
+    })
+  })
+
+  describe('Experiments Table Webview', () => {
+    const webview = new ExperimentsWebview('experiments')
+
+    it('should load as an editor', async () => {
+      const workbench = await browser.getWorkbench()
+
+      await workbench.executeCommand('DVC: Show Experiments')
+
+      await webview.open()
+
+      await browser.waitUntil(async () => {
+        const table = await webview.table$
+        return table.isDisplayed()
+      })
+
+      expect(await webview.table$$).toHaveLength(1)
+
+      await webview.close()
+    })
+
+    it('should update with a new row for each checkpoint when an experiment is running', async () => {
+      const workbench = await browser.getWorkbench()
+      const epochs = 15
+      await workbench.executeCommand('DVC: Reset and Run Experiment')
+
+      await webview.open()
+
+      await browser.waitUntil(() => webview.expandAllRows())
+
+      const initialRows = await webview.row$$
+
+      expect(initialRows.length).toBeGreaterThanOrEqual(4)
+
+      await browser.waitUntil(
+        async () => {
+          await webview.expandAllRows()
+          const currentRows = await webview.row$$
+          return currentRows.length >= initialRows.length + epochs
+        },
+        { timeout: 120000 }
+      )
+
+      const finalRows = await webview.row$$
+
+      expect(finalRows.length).toStrictEqual(initialRows.length + epochs)
+      await webview.close()
+    }).timeout(180000)
+  })
+
+  describe('Plots Webview', () => {
+    before(async () => {
+      await closeAllEditors()
+    })
+
+    const webview = new PlotsWebview('plots')
+
+    it('should load the plots webview with non-empty plots', async () => {
+      const workbench = await browser.getWorkbench()
+      await workbench.executeCommand('DVC: Show Plots')
+
+      await webview.open()
+
+      await browser.waitUntil(async () => {
+        return (await webview.vegaVisualization$$.length) === 6
+      })
+
+      const plots = await webview.vegaVisualization$$
+
+      for (const plot of plots) {
+        await plot.scrollIntoView()
+        const plotNotEmpty = await webview.plotNotEmpty(plot)
+        expect(plotNotEmpty).toBe(true)
+      }
+
+      await webview.close()
+    })
+  })
+})

extension/src/test/e2e/pageObjects/baseWebview.ts

@@ -0,0 +1,38 @@
+import { BasePage, IPageDecorator, PageDecorator } from 'wdio-vscode-service'
+import { webview as webviewLocators } from './locators'
+import * as locators from './locators'
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface BaseWebview extends IPageDecorator<typeof webviewLocators> {}
+
+@PageDecorator(webviewLocators)
+export class BaseWebview extends BasePage<
+  typeof webviewLocators,
+  typeof locators
+> {
+  public locatorKey: 'webview' | 'experiments' | 'plots'
+
+  constructor(locatorKey: keyof typeof locators) {
+    super(locators)
+    this.locatorKey = locatorKey
+  }
+
+  public async open() {
+    const webviewContainer = await this.outerFrame$
+
+    await this.outerFrame$.waitForDisplayed()
+
+    await browser.switchToFrame(webviewContainer)
+    await this.innerFrame$.waitForDisplayed()
+    const webviewInner = await browser.findElement(
+      'css selector',
+      this.locators.innerFrame
+    )
+    return browser.switchToFrame(webviewInner)
+  }
+
+  public async close() {
+    await browser.switchToFrame(null)
+    await browser.switchToFrame(null)
+  }
+}

extension/src/test/e2e/pageObjects/experimentsWebview.ts

@@ -0,0 +1,18 @@
+import { IPageDecorator, PageDecorator } from 'wdio-vscode-service'
+import { BaseWebview } from './baseWebview'
+import { experiments } from './locators'
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface ExperimentsWebview
+  extends IPageDecorator<typeof experiments> {}
+
+@PageDecorator(experiments)
+export class ExperimentsWebview extends BaseWebview {
+  public async expandAllRows() {
+    const expandRowButtons = await this.expandRowButton$$
+    for (const button of expandRowButtons) {
+      button.click()
+    }
+    return expandRowButtons.length === 0
+  }
+}

extension/src/test/e2e/pageObjects/locators.ts

@@ -0,0 +1,16 @@
+export const webview = {
+  innerFrame: '#active-frame',
+  outerFrame: '.webview.ready'
+}
+
+export const experiments = {
+  ...webview,
+  expandRowButton: 'button[title="Expand Row"]',
+  row: '[role=row]',
+  table: '[role=table]'
+}
+
+export const plots = {
+  ...webview,
+  vegaVisualization: 'div[aria-label="Vega visualization"]'
+}

extension/src/test/e2e/pageObjects/plotsWebview.ts

@@ -0,0 +1,21 @@
+import { IPageDecorator, PageDecorator } from 'wdio-vscode-service'
+import { BaseWebview } from './baseWebview'
+import { plots } from './locators'
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface PlotsWebview extends IPageDecorator<typeof plots> {}
+
+@PageDecorator(plots)
+export class PlotsWebview extends BaseWebview {
+  public async plotNotEmpty(plot: WebdriverIO.Element) {
+    return (await this.plotHasRects(plot)) || (await this.plotHasLines(plot))
+  }
+
+  private async plotHasRects(plot: WebdriverIO.Element) {
+    return (await plot.$$('[aria-roledescription="rect mark"]').length) > 0
+  }
+
+  private async plotHasLines(plot: WebdriverIO.Element) {
+    return (await plot.$$('[aria-roledescription="line mark"]').length) > 0
+  }
+}

extension/src/test/e2e/util.ts

@@ -0,0 +1,63 @@
+import { ViewControl } from 'wdio-vscode-service'
+
+export const dismissAllNotifications = () =>
+  browser.waitUntil(async () => {
+    const workbench = await browser.getWorkbench()
+    const notifications = await workbench.getNotifications()
+    for (const n of notifications) {
+      await n.dismiss()
+    }
+    const openNotifications = await workbench.getNotifications()
+    return openNotifications.length === 0
+  })
+
+export const getDVCActivityBarIcon = async (): Promise<ViewControl> => {
+  const workbench = await browser.getWorkbench()
+
+  const activityBar = workbench.getActivityBar()
+
+  await browser.waitUntil(
+    async () => !!(await activityBar.getViewControl('DVC'))
+  )
+  return activityBar.getViewControl('DVC') as Promise<ViewControl>
+}
+
+export const waitForViewContainerToLoad = async () => {
+  const initialProgressBars = await $$('.monaco-progress-container')
+  await browser.waitUntil(async () => {
+    const dvcIcon = await getDVCActivityBarIcon()
+    if (!dvcIcon) {
+      return false
+    }
+
+    const view = await dvcIcon.openView()
+
+    return !!view
+  })
+
+  return browser.waitUntil(async () => {
+    const numberOfProgressBarsInContainer = 7
+    const currentProgressBars = await $$('.monaco-progress-container')
+
+    if (
+      currentProgressBars.length <
+      initialProgressBars.length + numberOfProgressBarsInContainer
+    ) {
+      return false
+    }
+
+    for (const progress of currentProgressBars) {
+      if ((await progress.getAttribute('aria-hidden')) !== 'true') {
+        return false
+      }
+    }
+
+    return true
+  })
+}
+
+export const closeAllEditors = async () => {
+  const workbench = await browser.getWorkbench()
+  const editorView = workbench.getEditorView()
+  return editorView.closeAllEditors()
+}

extension/src/test/e2e/wdio.conf.ts

@@ -0,0 +1,41 @@
+import { join, resolve } from 'path'
+import { Options } from '@wdio/types'
+
+export const config: Options.Testrunner = {
+  after: async function () {
+    await browser.switchToFrame(null)
+    await browser.switchToFrame(null)
+  },
+  baseUrl: 'http://localhost',
+  before: async function () {
+    await browser.setWindowSize(1600, 1200)
+  },
+  capabilities: [
+    {
+      browserName: 'vscode',
+      browserVersion: 'insiders',
+      // @ts-expect-error these caps are not typed in WebdriverIO
+      'wdio:vscodeOptions': {
+        extensionPath: resolve(__dirname, '..', '..', '..'),
+        verboseLogging: false,
+        workspacePath: resolve(__dirname, '..', '..', '..', '..', 'demo')
+      }
+    }
+  ],
+  connectionRetryCount: 3,
+  connectionRetryTimeout: 120000,
+  framework: 'mocha',
+  maxInstances: 1,
+  mochaOpts: {
+    bail: true,
+    parallel: false,
+    retries: 0,
+    timeout: 60000,
+    ui: 'bdd'
+  },
+  outputDir: join(__dirname, 'logs'),
+  reporters: ['spec'],
+  services: ['vscode'],
+  specs: ['./src/test/e2e/*.test.ts'],
+  waitforTimeout: 10000
+}

package.json

@@ -7,7 +7,8 @@
   "scripts": {
     "format": "yarn turbo run format --parallel",
     "lint": "yarn turbo run lint --parallel",
-    "test:vscode": "yarn turbo run test-vscode-run",
+    "test:e2e": "yarn setup:venv && yarn turbo run test-e2e",
+    "test:vscode": "yarn turbo run test-vscode",
     "test:jest": "yarn turbo run test --parallel",
     "test": "run-p test:*",
     "cover:clean": "rm -rf ./coverage/combined && rm -rf ./extension/coverage rm -rf ./webview/coverage",

turbo.json

@@ -23,18 +23,21 @@
     },
     "cover-vscode-run": {
       "dependsOn": [
-        "dvc#test-vscode-build",
+        "dvc#test-build",
         "dvc#test",
         "dvc-vscode-webview#build",
         "dvc-vscode-webview#test"
       ],
       "outputs": ["coverage/integration/**"]
     },
-    "test-vscode-build": {
+    "test-build": {
       "outputs": ["dist/**"]
     },
-    "test-vscode-run": {
-      "dependsOn": ["dvc-vscode-webview#build", "dvc#test-vscode-build"]
+    "test-vscode": {
+      "dependsOn": ["dvc-vscode-webview#build", "dvc#test-build"]
+    },
+    "test-e2e": {
+      "dependsOn": ["dvc-vscode-webview#build", "dvc#test-build"]
     },
     "format": {
       "dependsOn": ["^format"],