From 6015d612f8d7c496dc38d40b79cdaacd83bee48d Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 20 Mar 2024 18:49:12 +0900 Subject: [PATCH] test: fix e2e + add ecosystem-ci test script (#311) * chore: add ecosystem-ci scripts * chore: lint * chore: lint * chore: fix escape * chore: handle package overrides * chore: tweak * fix: fix script * fix: more imba * fix: --no-frozen-lockfile * fix: npm overrides * chore: run e2e * ci: fix ci * ci: typo * ci: fix cd? * ci: tweak * chore: simplify xvfb-run * test: use playwright assertion * chore: simplify xvfb-run * chore: readme --- .github/workflows/build.yml | 18 ++++++--------- package.json | 5 +++- pnpm-lock.yaml | 17 +++++++++++--- scripts/ecosystem-ci.mts | 46 +++++++++++++++++++++++++++++++++++++ test-e2e/README.md | 13 +++++++++++ test-e2e/basic.test.ts | 16 +++++++------ test-e2e/helper.ts | 4 ++-- 7 files changed, 95 insertions(+), 24 deletions(-) create mode 100644 scripts/ecosystem-ci.mts create mode 100644 test-e2e/README.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8cc4379..f5ec60c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,19 +29,15 @@ jobs: - run: pnpm typecheck - run: pnpm lint - # see: https://github.com/microsoft/vscode-test/blob/main/sample/azure-pipelines.yml - - name: xvfb + - name: test run: | - /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & - echo ">>> Started xvfb" + xvfb-run pnpm test - - run: pnpm test - env: - DISPLAY: ':99' - - - run: pnpm test-e2e --retry 2 - env: - DISPLAY: ':99' + - name: test-e2e + run: | + pnpm -C samples/e2e i + npm -C samples/imba i + xvfb-run pnpm test-e2e --retry 2 - uses: actions/upload-artifact@v3 if: always() diff --git a/package.json b/package.json index b8e0eddf..12e6d6a9 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,8 @@ "test": "vscode-test", "test:watch": "vscode-test --watch-files src/**/*.ts --watch-files test/**/*.test.ts", "test-e2e": "vitest --root test-e2e", + "ecosystem-ci:build": "pnpm compile", + "ecosystem-ci:test": "tsx ./scripts/ecosystem-ci.mts", "typecheck": "tsc -b ./ ./test ./test-e2e", "lint": "eslint --cache .", "lint:fix": "eslint --cache --fix ." @@ -115,6 +117,7 @@ "@antfu/eslint-config": "^2.6.4", "@babel/parser": "^7.20.15", "@babel/types": "^7.20.7", + "@playwright/test": "^1.42.1", "@rauschma/stringio": "^1.4.0", "@types/fs-extra": "^9.0.13", "@types/glob": "^7.2.0", @@ -135,6 +138,7 @@ "chai": "^5.1.0", "changelogithub": "^0.13.3", "eslint": "^8.56.0", + "execa": "^8.0.1", "fast-glob": "^3.3.2", "flatted": "^3.2.9", "fs-extra": "^10.0.1", @@ -146,7 +150,6 @@ "mocha": "^10.3.0", "pathe": "^1.1.2", "picocolors": "^1.0.0", - "playwright": "^1.42.1", "prompts": "^2.4.2", "semver": "^7.3.5", "strip-ansi": "^7.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32b17885..6f1c8a9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ devDependencies: '@babel/types': specifier: ^7.20.7 version: 7.23.9 + '@playwright/test': + specifier: ^1.42.1 + version: 1.42.1 '@rauschma/stringio': specifier: ^1.4.0 version: 1.4.0 @@ -74,6 +77,9 @@ devDependencies: eslint: specifier: ^8.56.0 version: 8.56.0 + execa: + specifier: ^8.0.1 + version: 8.0.1 fast-glob: specifier: ^3.3.2 version: 3.3.2 @@ -107,9 +113,6 @@ devDependencies: picocolors: specifier: ^1.0.0 version: 1.0.0 - playwright: - specifier: ^1.42.1 - version: 1.42.1 prompts: specifier: ^2.4.2 version: 2.4.2 @@ -661,6 +664,14 @@ packages: dev: true optional: true + /@playwright/test@1.42.1: + resolution: {integrity: sha512-Gq9rmS54mjBL/7/MvBaNOBwbfnh7beHvS6oS4srqXFcQHpQCV1+c8JXWE8VLPyRDhgS3H8x8A7hztqI9VnwrAQ==} + engines: {node: '>=16'} + hasBin: true + dependencies: + playwright: 1.42.1 + dev: true + /@rauschma/stringio@1.4.0: resolution: {integrity: sha512-3uor2f/MXZkmX5RJf8r+OC3WvZVzpSme0yyL0rQDPEnatE02qRcqwEwnsgpgriEck0S/n4vWtUd6tTtrJwk45Q==} dependencies: diff --git a/scripts/ecosystem-ci.mts b/scripts/ecosystem-ci.mts new file mode 100644 index 00000000..c016d7dd --- /dev/null +++ b/scripts/ecosystem-ci.mts @@ -0,0 +1,46 @@ +import process from 'node:process' +import fs from 'node:fs' +import { $ as $_ } from 'execa' + +const $ = $_({ stdio: 'inherit', verbose: true }) + +async function main() { + // setup pakcage overrides for samples used by test-e2e + const pkg = await readJson('package.json') + if (pkg.pnpm?.overrides) { + // pnpm + await editJson('samples/e2e/package.json', (pkg2) => { + pkg2.pnpm = pkg.pnpm + return pkg2 + }) + await $({ cwd: 'samples/e2e' })`pnpm i --no-frozen-lockfile` + + // npm + await editJson('samples/imba/package.json', (pkg2) => { + pkg2.devDependencies.vitest = pkg.pnpm.overrides.vitest + pkg2.overrides = pkg.pnpm.overrides + return pkg2 + }) + await $({ cwd: 'samples/imba' })`npm i` + } + + if (process.env.CI === 'true' && process.platform === 'linux') { + await $`xvfb-run pnpm test` + await $`xvfb-run pnpm test-e2e --retry 2` + } + else { + await $`pnpm test` + await $`pnpm test-e2e` + } +} + +async function readJson(file: string) { + return JSON.parse(await fs.promises.readFile(file, 'utf-8')) +} + +async function editJson(file: string, edit: (data: any) => any) { + const data = await readJson(file) + await fs.promises.writeFile(file, JSON.stringify(edit(data), null, 2)) +} + +main() diff --git a/test-e2e/README.md b/test-e2e/README.md new file mode 100644 index 00000000..fc539ed4 --- /dev/null +++ b/test-e2e/README.md @@ -0,0 +1,13 @@ +# test-e2e + +To open playwright inspector, either run with `PWDEBUG` + +```sh +PWDEBUG=1 pnpm test-e2e +``` + +or use `page.pause` inside a test code + +```ts +await page.pase() +``` diff --git a/test-e2e/basic.test.ts b/test-e2e/basic.test.ts index fbe8e72d..dd5f7366 100644 --- a/test-e2e/basic.test.ts +++ b/test-e2e/basic.test.ts @@ -1,4 +1,5 @@ import { beforeAll } from 'vitest' +import { expect } from '@playwright/test' import { test } from './helper' // Vitst extension doesn't work with CI flag @@ -23,10 +24,10 @@ test('basic', async ({ launch }) => { await page.getByRole('button', { name: 'Run Tests' }).click() // check results - await page.locator(`[title*="pass.test.ts (Passed)"]`).click() - await page.locator(`[title*="fail.test.ts (Failed)"]`).click() - await page.locator(`[title*="mix.test.ts (Failed)"]`).click() - await page.locator(`[title*="3/7 tests passed"]`).click() + await expect(page.locator(`[title*="3/7 tests passed"]`)).toBeVisible() + await expect(page.locator(`[title*="pass.test.ts (Passed)"]`)).toBeVisible() + await expect(page.locator(`[title*="fail.test.ts (Failed)"]`)).toBeVisible() + await expect(page.locator(`[title*="mix.test.ts (Failed)"]`)).toBeVisible() }) test('imba', async ({ launch }) => { @@ -46,7 +47,8 @@ test('imba', async ({ launch }) => { await page.getByRole('button', { name: 'Run Tests' }).click() // check results - await page.locator(`[title*="basic.test.imba (Passed)"]`).click() - await page.locator(`[title*="utils.imba (Passed)"]`).click() - await page.locator(`[title*="counter.imba (Failed)"]`).click() + await expect(page.locator(`[title*="5/7 tests passed"]`)).toBeVisible() + await expect(page.locator(`[title*="basic.test.imba (Passed)"]`)).toBeVisible() + await expect(page.locator(`[title*="utils.imba (Passed)"]`)).toBeVisible() + await expect(page.locator(`[title*="counter.imba (Failed)"]`)).toBeVisible() }) diff --git a/test-e2e/helper.ts b/test-e2e/helper.ts index b7a5dcab..50f2703c 100644 --- a/test-e2e/helper.ts +++ b/test-e2e/helper.ts @@ -2,8 +2,8 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { download } from '@vscode/test-electron' -import { _electron } from 'playwright' -import type { Page } from 'playwright' +import { _electron } from '@playwright/test' +import type { Page } from '@playwright/test' import { test as baseTest } from 'vitest' // based on