Visual regression testing / check if jdaviz application renders with jupyter lab 3.0

.github/workflows/ci_visual.yml

@@ -0,0 +1,103 @@
+name: Visual regression test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: '*'
+  schedule:
+    # run at 9am UTC on Mondays
+    - cron: '0 9 * * 1'
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Setup conda
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: jdaviz-test
+        environment-file: test-environment.yml
+        python-version: ${{ matrix.python-version }}
+        mamba-version: "*"
+        auto-activate-base: false
+        channels: conda-forge
+
+    - name: Install the package
+      run: |
+        pip install .
+
+    - name: Build Python package
+      run: |
+        python setup.py sdist bdist_wheel
+        cd dist
+        sha256sum * | tee SHA256SUMS
+
+    - name: Upload builds
+      uses: actions/upload-artifact@v2
+      with:
+        name: dist ${{ github.run_number }}
+        path: ./dist
+
+  visual-regression-tests:
+    runs-on: ubuntu-latest
+    needs: [build]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: jdaviz-test
+          environment-file: test-environment.yml
+          python-version: 3.9
+          mamba-version: "*"
+          auto-activate-base: false
+          channels: conda-forge
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: dist ${{ github.run_number }}
+          path: ./dist
+
+      - name: Install the package
+        run: pip install -vv jdaviz*.whl
+        working-directory: dist
+
+      - name: Install Galata
+        run: yarn install
+        working-directory: ui-tests
+
+      - name: Launch JupyterLab
+        run: yarn run start-jlab:detached
+        working-directory: ui-tests
+
+      - name: Wait for JupyterLab
+        uses: ifaxity/wait-on-action@v1
+        with:
+          resource: http-get://localhost:8999/api
+          timeout: 20000
+
+      - name: Run UI Tests
+        run: yarn run test
+        working-directory: ui-tests
+
+      - name: Upload UI Test artifacts
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ui-test-output
+          path: ui-tests/test-output
+

setup.cfg

@@ -24,8 +24,9 @@ install_requires =
     glue-jupyter>=0.2.2
     echo>=0.5.0
     ipyvue>=1.4.1
+    ipyvolume>=0.6.0a8
     ipyvuetify>=1.5.1
-    ipysplitpanes>=0.1.0
+    ipysplitpanes>=0.2.0
     ipygoldenlayout>=0.3.0
     voila>=0.2.4
     pyyaml>=5.4.1

test-environment.yml

@@ -0,0 +1,10 @@
+name: jdaviz-test
+channels:
+    - conda-forge
+dependencies:
+    - pip
+    - python
+    - yarn
+    - nodejs=12
+    - jupyterlab=3.0.3  # to build the lab federated bundle
+    - jupyter-packaging # to build the wheel

ui-tests/README.md

@@ -0,0 +1 @@
+Should be run the same as https://github.com/bqplot/bqplot/tree/master/ui-tests

ui-tests/galata-config.json

@@ -0,0 +1,3 @@
+{
+  "testId": "test"
+}

ui-tests/jupyter_server_config.py

@@ -0,0 +1,7 @@
+c.ServerApp.port = 8999
+c.ServerApp.token = ""
+c.ServerApp.password = ""
+c.ServerApp.disable_check_xsrf = True
+c.ServerApp.open_browser = False
+c.LabApp.open_browser = False
+c.LabApp.expose_app_in_browser = True

ui-tests/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "jdaviz-ui-tests",
+  "version": "1.0.0",
+  "description": "jdaviz UI Tests",
+  "private": true,
+  "scripts": {
+    "start-jlab": "jupyter lab --config ./jupyter_server_config.py",
+    "start-jlab:detached": "yarn run start-jlab&",
+    "clean": "rimraf tests/notebooks/.ipynb_checkpoints && rimraf test-output",
+    "test": "yarn run clean && galata --jlab-base-url http://localhost:8999 --image-match-threshold 0.25",
+    "update-references": "galata --update-references"
+  },
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@jupyterlab/galata": "3.0.3-3",
+    "klaw-sync": "^6.0.0",
+    "rimraf": "^3.0.2"
+  }
+}

ui-tests/tests/jdaviz.test.ts

@@ -0,0 +1,149 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+import { galata, describe, test } from '@jupyterlab/galata';
+import * as path from 'path';
+const klaw = require('klaw-sync');
+
+jest.setTimeout(600000);
+
+const filterUpdateNotebooks = item => {
+  const basename = path.basename(item.path);
+  return basename.includes('_update');
+}
+
+describe('jdaviz Visual Regression', () => {
+  beforeAll(async () => {
+    await galata.resetUI();
+  });
+
+  afterAll(async () => {
+    galata.context.capturePrefix = '';
+  });
+
+  test('Upload files to JupyterLab', async () => {
+    await galata.contents.moveDirectoryToServer(
+      path.resolve(__dirname, `./notebooks`),
+      'uploaded'
+    );
+    expect(
+      await galata.contents.fileExists('uploaded/app.ipynb')
+    ).toBeTruthy();
+  });
+
+  test('Refresh File Browser', async () => {
+    await galata.filebrowser.refresh();
+  });
+
+  test('Open directory uploaded', async () => {
+    await galata.filebrowser.openDirectory('uploaded');
+    expect(
+      await galata.filebrowser.isFileListedInBrowser('app.ipynb')
+    ).toBeTruthy();
+  });
+
+  test('Check jdaviz first renders', async () => {
+    const paths = klaw('tests/notebooks', {filter: item => !filterUpdateNotebooks(item), nodir: true});
+    const notebooks = paths.map(item => path.basename(item.path));
+
+    let results = [];
+
+    for (const notebook of notebooks) {
+      galata.context.capturePrefix = notebook;
+
+      await galata.notebook.open(notebook);
+      expect(await galata.notebook.isOpen(notebook)).toBeTruthy();
+      await galata.notebook.activate(notebook);
+      expect(await galata.notebook.isActive(notebook)).toBeTruthy();
+
+      let numCellImages = 0;
+
+      const getCaptureImageName = (id: number): string => {
+        return `cell-${id}`;
+      };
+
+      await galata.notebook.runCellByCell({
+        onAfterCellRun: async (cellIndex: number) => {
+          const cell = await galata.notebook.getCellOutput(cellIndex);
+          if (cell) {
+            if (
+              await galata.capture.screenshot(
+                getCaptureImageName(numCellImages),
+                cell
+              )
+            ) {
+              numCellImages++;
+            }
+          }
+        }
+      });
+
+      for (let c = 0; c < numCellImages; ++c) {
+        results.push(await galata.capture.compareScreenshot(getCaptureImageName(c)));
+      }
+
+      await galata.notebook.close(true);
+    }
+
+    for (const result of results) {
+      expect(result).toBe('same');
+    }
+  });
+
+  test('Check jdaviz update plot properties', async () => {
+    const paths = klaw('tests/notebooks', {filter: item => filterUpdateNotebooks(item), nodir: true});
+    const notebooks = paths.map(item => path.basename(item.path));
+
+    let results = [];
+
+    for (const notebook of notebooks) {
+      galata.context.capturePrefix = notebook;
+
+      await galata.notebook.open(notebook);
+      expect(await galata.notebook.isOpen(notebook)).toBeTruthy();
+      await galata.notebook.activate(notebook);
+      expect(await galata.notebook.isActive(notebook)).toBeTruthy();
+
+      let numCellImages = 0;
+
+      const getCaptureImageName = (id: number): string => {
+        return `cell-${id}`;
+      };
+
+      await galata.notebook.runCellByCell({
+        onAfterCellRun: async (cellIndex: number) => {
+          // Always get first cell output which must contain the plot
+          const cell = await galata.notebook.getCellOutput(0);
+          if (cell) {
+            if (
+              await galata.capture.screenshot(
+                getCaptureImageName(numCellImages),
+                cell
+              )
+            ) {
+              numCellImages++;
+            }
+          }
+        }
+      });
+
+      for (let c = 0; c < numCellImages; ++c) {
+        results.push(await galata.capture.compareScreenshot(getCaptureImageName(c)));
+      }
+
+      await galata.notebook.close(true);
+    }
+
+    for (const result of results) {
+      expect(result).toBe('same');
+    }
+  });
+
+  test('Open home directory', async () => {
+    await galata.filebrowser.openHomeDirectory();
+  });
+
+  test('Delete uploaded directory', async () => {
+    await galata.contents.deleteDirectory('uploaded');
+  });
+});

ui-tests/tests/notebooks/app.ipynb

@@ -0,0 +1,45 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa840ee6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import warnings\n",
+    "warnings.simplefilter('ignore')\n",
+    "import jdaviz\n",
+    "jdaviz.Application()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.10"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}