Setup PlayWright tests (#38)

- First setup of e2e testing with playwright including github workflow
- Added dependency on crossmodel-browser-app in e2e-tests
- Updated devcontainer with Dockerfile and updated README
- Updated decontainer config to support WSLg
- Added extra yarn execution before installing playwright deps
- Added allure reporting for ci runs and added dependencies

- First step for using tabbar toolbar in playwright tests
- Implement Theia Tab bar toolbar and Theia single input dialog page object
- Create tests for tab bar toolbar actions (entity, relationship & diagram)

.devcontainer/Dockerfile

@@ -0,0 +1,10 @@
+# We use the Node-16 on Debian Bullseye.
+ARG VARIANT="16-bullseye"
+FROM mcr.microsoft.com/devcontainers/typescript-node:0-${VARIANT}
+
+# Install OS packages needed for building Theia.
+RUN apt-get update \
+    && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends \
+    libsecret-1-dev \
+    libxkbfile-dev

.devcontainer/devcontainer.json

@@ -1,18 +1,55 @@
 // Dev-Container for CrossModel.
-// For format details, see https://aka.ms/devcontainer.json.
 {
   "name": "Node.js & TypeScript",
-  // https://github.com/devcontainers/templates/tree/main/src/typescript-node
-  "image": "mcr.microsoft.com/devcontainers/typescript-node:1-16-bullseye",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
   "features": {
     "ghcr.io/devcontainers/features/python:1": {
       "version": "3.11.4"
     }
   },
-
+  "customizations": {
+    "vscode": {
+      // Install a list of extensions in the Dev Container so everything is reasdy to develop and test.
+      "extensions": [
+        "langium.langium-vscode",
+        "esbenp.prettier-vscode",
+        "eamodio.gitlens",
+        "davidanson.vscode-markdownlint",
+        "ms-playwright.playwright",
+        "dbaeumer.vscode-eslint",
+        "orta.vscode-jest"
+      ]
+    }
+  },
   // Allow port 3000 to be forwarded.
   "forwardPorts": [3000],
-
-  // Install the needed OS libs for building.
-  "postCreateCommand": "sudo apt-get update && sudo apt-get -y install libsecret-1-dev libxkbfile-dev"
+  "portsAttributes": {
+    "3000": {
+      "label": "Theia Backend"
+    }
+  },
+  // Install the playwright dependencies.
+  "postCreateCommand": "yarn && yarn playwright install --with-deps",
+  // When we want to be able to run playwright headed within the Dev Container,
+  // we need to setup the mounts and container environment variables like below.
+  "mounts": [
+    {
+      "source": "/run/desktop/mnt/host/wslg/.X11-unix",
+      "target": "/tmp/.X11-unix",
+      "type": "bind"
+    },
+    {
+      "source": "/run/desktop/mnt/host/wslg",
+      "target": "/mnt/wslg",
+      "type": "bind"
+    }
+  ],
+  "containerEnv": {
+    "DISPLAY": ":0",
+    "WAYLAND_DISPLAY": "wayland-0",
+    "XDG_RUNTIME_DIR": "/mnt/wslg/runtime-dir",
+    "PULSE_SERVER": "/mnt/wslg/PulseServer"
+  }
 }

.github/workflows/build-and-test.yml

@@ -46,43 +46,61 @@ jobs:
           yarn build
 
       # Execute the tests.
-      - name: Test
+      - name: Run Unit Tests
         run: yarn test
         env:
           # The test result file name can be controlled using the following environment variable.
           JEST_JUNIT_OUTPUT_NAME: unit-test-results-${{ runner.os }}.xml
 
-      # Upload Test Results (The different files for the OSes will end up in the same artifact).
-      - name: Upload Test Results
+      # Upload Unit Test Results (The different files for the OSes will end up in the same artifact).
+      - name: Upload Unit Test Results
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: unit-test-tesults
           # Include the unit-test-results folders (which is in the root of the workspace).
           path: unit-test-results
+          retention-days: 30
+
+      # Run PlayWright tests, only on Linux container.
+      - name: Install Playwright Browsers
+        if: runner.os == 'Linux'
+        run: yarn --cwd ./e2e-tests/ playwright:install
+      - name: Run Playwright tests
+        if: runner.os == 'Linux'
+        uses: coactions/setup-xvfb@v1
+        with:
+          run: yarn ui-test
+      - name: Upload PlayWrite test report
+        if: always() && runner.os == 'Linux'
+        uses: actions/upload-artifact@v3
+        with:
+          name: allure-results
+          path: e2e-tests/allure-results/
+          retention-days: 30
 
       # Run lint only on Linux (since it only makes sense to run it once, and linux is the fastest).
       - name: Lint
-        if: runner.os == 'Linux'
+        if: always() && runner.os == 'Linux'
         run: yarn lint
 
-  # Publish a test report using the test result files published in the previous step (executed per OS).
-  publish-test-report:
-    name: Publish Test Report
+  # Publish a test report using the unit test result files published in the previous step (which was executed per OS).
+  publish-unit-test-report:
+    name: Publish Unit Test Report
     needs: build-and-test
     runs-on: ubuntu-latest
     if: always()
     steps:
       # Download the test results artifacts.
-      - name: Download Test Results
+      - name: Download Unit Test Results
         uses: actions/download-artifact@v3
         with:
           name: unit-test-tesults
           path: unit-test-tesults
       # Publish Test Results
-      - name: Publish Test Results
+      - name: Publish Unit Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
         with:
-          check_name: Test Results
+          check_name: Unit Test Results
           files: |
             unit-test-tesults/**/*.xml

.github/workflows/cicd-main.yml

@@ -8,3 +8,55 @@ on:
 jobs:
   build-and-test:
     uses: ./.github/workflows/build-and-test.yml
+  # Publish a test report using the playwright result files published in the previous step (execute in Linux only).
+  publish-playwright-test-report:
+    name: Publish PlayWright Test Report
+    needs: build-and-test
+    permissions:
+      pages: write # to deploy to Pages
+      id-token: write # to verify the deployment originates from an appropriate source
+    # Deploy to the github-pages environment
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      # Setup the GitHub Pages, if it doesn't exist yet.
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      # Download the test results artifacts.
+      - name: Download Test Results
+        uses: actions/download-artifact@v3
+        with:
+          name: allure-results
+          path: allure-results
+      # Get the gh-pages history, so the next report can be generated with history.
+      - name: Get History
+        uses: actions/checkout@v4
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages
+      # Generate the Allure Report
+      - name: Generate Allure Report
+        uses: simple-elf/allure-report-action@master
+        with:
+          # Where to find the allure results.
+          allure_results: allure-results
+          # Where to publish the history.
+          allure_history: allure-history
+          keep_reports: 100
+          # Subfolder in the destination.
+          subfolder: allure
+          # Where to find the gh-pages history.
+          gh_pages: gh-pages
+      # Upload allure-history report to github-pages artifact.
+      - name: Upload Pages
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: 'allure-history'
+      # Deploy the github-pages artifact to GitHub pages.
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.gitignore

@@ -14,5 +14,8 @@ bin
 .ignored_out
 dist
 *.port
-**/unit-test-results
+**/*test-results
 **/coverage
+**/playwright-report
+**/playwright/.cache
+**/allure-results

README.md

@@ -24,7 +24,7 @@ _or:_ launch `Launch CrossModel Electron` configuration from VS code.
 
 ## Example Workspace
 
-Under `examples/workspace` we provide an example workspace with some demo packages containing entities, relationships and system diagrams.
+Under `examples/verdaccio-example/workspace` we provide an example workspace with some demo packages containing entities, relationships and system diagrams.
 Each package represents a dedicated system or library and may depend on other packages.
 
 Using a known package structure - npm in our case - we can re-use large parts of the package management to download dependencies that are not locally available from an external package registry.
@@ -35,7 +35,7 @@ You can start verdaccio using
 
 The local npm registry will be available under `http://localhost:4873/` where we already provide four packages by default.
 
-After opening the workspace, you can install the necessary dependencies in the example workspace by opening a terminal in `examples/workspace` and execute
+After opening the workspace, you can install the necessary dependencies in the example workspace by opening a terminal in `examples/verdaccio-example/workspace` and execute
 
     npm install
 
@@ -65,6 +65,10 @@ Any code changes will be automatically detected and the application will be re-c
 If you only made changes to the frontend or plugins, simply reloading the running application with `F5` is enough.
 If you also made changes to the backend, you can close and restart the appliaction without manual re-compilation.
 
+### Developing in Dev Container
+
+You can work on CrossModel from within a Dev Container. The best way to do so is create a new Dev Container and cloning the repository in there. For instructions please consult [this](https://code.visualstudio.com/docs/devcontainers/containers#_quick-start-open-a-git-repository-or-github-pr-in-an-isolated-container-volume) page.
+
 ## Packaging
 
 To package the application use

e2e-tests/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "crossmodel-e2e-tests",
+  "version": "0.0.0",
+  "private": true,
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "CrossBreeze",
+    "email": "devops@crossbreeze.nl"
+  },
+  "scripts": {
+    "build": "tsc -b && npx playwright install chromium",
+    "clean": "rimraf lib tsconfig.tsbuildinfo",
+    "lint": "eslint -c ../.eslintrc.js --ext .ts ./src",
+    "playwright:install": "yarn playwright install --with-deps",
+    "prepare": "yarn clean && yarn build && yarn lint",
+    "test": "yarn playwright test"
+  },
+  "dependencies": {
+    "@playwright/test": "^1.37.1",
+    "@theia/playwright": "1.43.1"
+  },
+  "devDependencies": {
+    "allure-playwright": "^2.9.2"
+  }
+}

e2e-tests/playwright.config.ts

@@ -0,0 +1,33 @@
+/********************************************************************************
+ * Copyright (c) 2023 CrossBreeze.
+ ********************************************************************************/
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+    testDir: './src/tests',
+    testMatch: ['**/*.spec.ts'],
+    workers: process.env.CI ? 1 : 2,
+    retries: process.env.CI ? 1 : 0,
+    // The number of times to repeat each test, useful for debugging flaky tests
+    repeatEach: 1,
+    // Timeout for each test in milliseconds: 60 seconds.
+    timeout: 60 * 1000,
+    use: {
+        baseURL: 'http://localhost:3000',
+        browserName: 'chromium',
+        screenshot: 'only-on-failure',
+        viewport: { width: 1920, height: 1080 }
+    },
+    snapshotDir: './src/tests/snapshots',
+    expect: {
+        toMatchSnapshot: { threshold: 0.01 }
+    },
+    preserveOutput: 'failures-only',
+    reporter: process.env.CI ? [['list'], ['allure-playwright'], ['github']] : [['list'], ['html']],
+    /* Run your local dev server before starting the tests */
+    webServer: {
+        command: 'yarn --cwd ../../ start:browser',
+        url: 'http://localhost:3000',
+        reuseExistingServer: !process.env.CI
+    }
+});

e2e-tests/src/fixtures/crossmodel-fixture.ts

@@ -0,0 +1,17 @@
+/********************************************************************************
+ * Copyright (c) 2023 CrossBreeze.
+ ********************************************************************************/
+import { test, Page } from '@playwright/test';
+import { CrossModelApp } from '../page-objects/crossmodel-app';
+import { CrossModelWorkspace } from '../page-objects/crossmodel-workspace';
+
+export let page: Page;
+export let app: CrossModelApp;
+
+test.beforeAll(async ({ browser }) => {
+    page = await browser.newPage();
+    const ws = new CrossModelWorkspace(['src/resources/sample-workspace']);
+    app = await CrossModelApp.load(page, ws);
+});
+
+export default test;

e2e-tests/src/page-objects/crossmodel-app.ts

@@ -0,0 +1,6 @@
+/********************************************************************************
+ * Copyright (c) 2023 CrossBreeze.
+ ********************************************************************************/
+import { TheiaApp } from '@theia/playwright';
+
+export class CrossModelApp extends TheiaApp {}

e2e-tests/src/page-objects/crossmodel-explorer-view.ts

@@ -0,0 +1,14 @@
+/********************************************************************************
+ * Copyright (c) 2023 CrossBreeze.
+ ********************************************************************************/
+import { TheiaApp, TheiaExplorerView } from '@theia/playwright';
+import { TheiaTabBarToolbar } from './theia-tabbar-toolbar';
+
+export class CrossModelExplorerView extends TheiaExplorerView {
+    public readonly tabBarToolbar: TheiaTabBarToolbar;
+
+    constructor(app: TheiaApp) {
+        super(app);
+        this.tabBarToolbar = new TheiaTabBarToolbar(this);
+    }
+}

e2e-tests/src/page-objects/crossmodel-workspace.ts

@@ -0,0 +1,6 @@
+/********************************************************************************
+ * Copyright (c) 2023 CrossBreeze.
+ ********************************************************************************/
+import { TheiaWorkspace } from '@theia/playwright';
+
+export class CrossModelWorkspace extends TheiaWorkspace {}

e2e-tests/src/page-objects/theia-single-input-dialog.ts

@@ -0,0 +1,20 @@
+/********************************************************************************
+ * Copyright (c) 2023 CrossBreeze.
+ ********************************************************************************/
+import { OSUtil, TheiaDialog, USER_KEY_TYPING_DELAY } from '@theia/playwright';
+
+export class TheiaSingleInputDialog extends TheiaDialog {
+    async enterSingleInput(inputValue: string): Promise<void> {
+        const inputField = await this.page.waitForSelector(`${this.blockSelector} .theia-input`);
+        await inputField.press(OSUtil.isMacOS ? 'Meta+a' : 'Control+a');
+        await inputField.fill(inputValue);
+        await this.page.waitForTimeout(USER_KEY_TYPING_DELAY);
+    }
+
+    async confirm(): Promise<void> {
+        if (!(await this.validationResult())) {
+            throw new Error(`Unexpected validation error in TheiaSingleInputDialog: '${await this.getValidationText()}`);
+        }
+        await this.clickMainButton();
+    }
+}

e2e-tests/src/page-objects/theia-tabbar-toolbar.ts

@@ -0,0 +1,27 @@
+/********************************************************************************
+ * Copyright (c) 2023 CrossBreeze.
+ ********************************************************************************/
+import { TheiaToolbarItem, TheiaView } from '@theia/playwright';
+import { TheiaViewObject } from './theia-view-object';
+
+export class TheiaTabBarToolbar extends TheiaViewObject {
+    constructor(view: TheiaView) {
+        super(view, '.p-TabBar-toolbar');
+    }
+
+    async toolBarItem(commandId: string): Promise<TheiaToolbarItem | undefined> {
+        const toolbarHandle = await this.objectElementHandle();
+        if (!toolbarHandle) {
+            return undefined;
+        }
+        const item = await toolbarHandle.$(this.toolBarItemSelector(commandId));
+        if (item) {
+            return new TheiaToolbarItem(this.app, item);
+        }
+        return undefined;
+    }
+
+    protected toolBarItemSelector(toolbarItemId = ''): string {
+        return `div.item > div${toolbarItemId ? `[id="${toolbarItemId}"]` : ''}`;
+    }
+}