test: experiment with full e2e tests

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ env:
   FORCE_COLOR: 1
 
 jobs:
-  build:
+  test:
     runs-on: ${{ matrix.os }}
     name: Run tests on ${{ matrix.os }}, Node ${{ matrix.node-version }}
     strategy:
@@ -40,3 +40,33 @@ jobs:
         name: vsc-extension
         path: "*.vsix"
         retention-days: 30
+  test-e2e:
+    runs-on: ${{ matrix.os }}
+    name: Run e2e tests
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js
+      uses: actions/setup-node@v3
+      with:
+        node-version: 18
+    - run: npm ci
+    - run: npx playwright install --with-deps
+    - run: npm run build
+    - run: npm ci
+      working-directory: ./tests-integration
+    - run: xvfb-run npx playwright test
+      working-directory: ./tests-integration
+      if: matrix.os == 'ubuntu-latest'
+    - run: npx playwright test
+      working-directory: ./tests-integration
+      if: matrix.os != 'ubuntu-latest'
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: playwright-report-${{ matrix.os }}
+        path: tests-integration/playwright-report/
+        retention-days: 30

tests-integration/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "tests-integration",
+  "devDependencies": {
+    "@vscode/test-electron": "^2.3.3"
+  }
+}

tests-integration/playwright.config.ts

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { defineConfig } from '@playwright/test';
+import { TestOptions } from './tests/baseTest';
+
+export default defineConfig<void, TestOptions>({
+  reporter: process.env.CI ? 'html' : 'list',
+  timeout: 120_000,
+  workers: 1,
+  expect: {
+    timeout: 30_000,
+  },
+  projects: [
+    {
+      name: 'VSCode stable',
+      use: {
+        vscodeVersion: 'stable',
+      }
+    },
+    {
+      name: 'VSCode insiders',
+      use: {
+        vscodeVersion: 'insiders',
+      }
+    }
+  ]
+});