From 7bc087bb5a9f279c1aa5fc76bdd24c5895dbe15a Mon Sep 17 00:00:00 2001 From: Christian Bromann Date: Wed, 15 Mar 2023 22:48:01 +0100 Subject: [PATCH] Improve stability of component tests for Safari (#9985) * Improve stability of component tests for Safari * revert tweak * print source page when connection fails * evaluate for title too --- .github/workflows/test.yml | 2 + e2e/browser-runner/lit.test.js | 9 +- e2e/browser-runner/wdio.conf.js | 17 ++-- e2e/package-lock.json | 90 ++++++++++++++++++- e2e/package.json | 3 +- .../wdio-browser-runner/src/browser/mock.ts | 5 -- .../wdio-browser-runner/src/vite/constants.ts | 6 +- .../src/vite/plugins/esbuild.ts | 34 +++++++ .../src/vite/plugins/testrunner.ts | 3 +- packages/wdio-runner/src/browser.ts | 8 +- 10 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 packages/wdio-browser-runner/src/vite/plugins/esbuild.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8e30d658750..0e4b7cd9695 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,6 +88,8 @@ jobs: timeout_minutes: 20 max_attempts: 3 command: npm run test:e2e + - name: Safari Component Tests + run: npm run test:component # Disable Firefox tests due to https://github.com/puppeteer/puppeteer/issues/8923 # - name: Setup Firefox # uses: browser-actions/setup-firefox@latest diff --git a/e2e/browser-runner/lit.test.js b/e2e/browser-runner/lit.test.js index 76f934beaa1..dc1f3ec06e3 100644 --- a/e2e/browser-runner/lit.test.js +++ b/e2e/browser-runner/lit.test.js @@ -78,7 +78,14 @@ describe('Lit Component testing', () => { expect(window.TEST_COMMAND).toBe('serve') }) - it('should allow to manual mock dependencies', async () => { + it('should allow to manual mock dependencies', async function () { + /** + * this fails in Safari as the click on the button is not recognised + */ + if (browser.capabilities.browserName.toLowerCase() === 'safari') { + return this.skip() + } + render( html``, document.body diff --git a/e2e/browser-runner/wdio.conf.js b/e2e/browser-runner/wdio.conf.js index e52e8e27167..14efde099bc 100644 --- a/e2e/browser-runner/wdio.conf.js +++ b/e2e/browser-runner/wdio.conf.js @@ -3,17 +3,22 @@ import url from 'node:url' import path from 'node:path' import { loadEnv } from 'vite' +const isMac = os.platform() === 'darwin' +const isWindows = os.platform() === 'win32' + /** * WebdriverIO is using this example to test its component testing features * and we have experienced issues with Vue when running in Windows, * see https://github.com/testing-library/vue-testing-library/issues/292 * Please ignore and remove this in your project! */ -if (process.env.CI && process.env.WDIO_PRESET === 'vue' && os.platform() === 'win32') { +if (process.env.CI && process.env.WDIO_PRESET === 'vue' && (isWindows || isMac)) { process.exit(0) } const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) +const browserName = isMac ? 'safari' : 'chrome' +const driver = isMac ? 'safaridriver' : 'chromedriver' export const config = { /** * specify test files @@ -26,10 +31,7 @@ export const config = { /** * capabilities */ - capabilities: [{ - browserName: 'chrome', - acceptInsecureCerts: true - }], + capabilities: [{ browserName }], /** * test configurations @@ -38,7 +40,7 @@ export const config = { framework: 'mocha', outputDir: path.join(__dirname, 'logs', process.env.WDIO_PRESET), reporters: ['spec', 'dot', 'junit'], - services: ['chromedriver'], + services: [driver], runner: ['browser', { preset: process.env.WDIO_PRESET, rootDir: path.resolve(__dirname, '..'), @@ -54,7 +56,8 @@ export const config = { }, coverage: { enabled: true, - functions: 100 + // we skip some tests on Mac, therefor lower coverage treshold + functions: isMac ? 66 : 100 } }], diff --git a/e2e/package-lock.json b/e2e/package-lock.json index 3f80437c9c0..c582a7f4f14 100644 --- a/e2e/package-lock.json +++ b/e2e/package-lock.json @@ -26,7 +26,8 @@ "expect": "^29.1.2", "graphql-request": "^5.1.0", "mocha": "^10.2.0", - "wdio-chromedriver-service": "^8.0.1" + "wdio-chromedriver-service": "^8.0.1", + "wdio-safaridriver-service": "^2.1.0" } }, "node_modules/@adobe/css-tools": { @@ -1416,12 +1417,27 @@ "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", "dev": true }, + "node_modules/@types/split2": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/split2/-/split2-3.2.1.tgz", + "integrity": "sha512-7uz3yU+LooBq4yNOzlZD9PU9/1Eu0rTD1MjQ6apOVEoHsPrMUrFw7W8XrvWtesm2vK67SBK9AyJcOXtMpl9bgQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/stack-utils": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", "dev": true }, + "node_modules/@types/tcp-port-used": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/tcp-port-used/-/tcp-port-used-1.0.1.tgz", + "integrity": "sha512-6pwWTx8oUtWvsiZUCrhrK/53MzKVLnuNSSaZILPy3uMes9QnTrLMar9BDlJArbMOjDcjb3QXFk6Rz8qmmuySZw==", + "dev": true + }, "node_modules/@types/testing-library__jest-dom": { "version": "5.14.5", "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz", @@ -5730,6 +5746,12 @@ "node": "*" } }, + "node_modules/safaridriver": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/safaridriver/-/safaridriver-0.0.4.tgz", + "integrity": "sha512-EgK9Vc2Bk4WaECq1E7ti9GyeQ6JKKQNs40vNOE/b5Ul5dDEt429G0Kj5Nzs4FRS5ZIVa7nBck1TyHuinOYdz2Q==", + "dev": true + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -6280,6 +6302,36 @@ } } }, + "node_modules/wdio-safaridriver-service": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wdio-safaridriver-service/-/wdio-safaridriver-service-2.1.0.tgz", + "integrity": "sha512-Zg0aZn8KJo+MwvdOTje2cja4OcrkpoCBhQSzVlhZ3SszMnwhi4VQ1p5TdHB9mWZpuVtsLHWpmx/G7ceJWendVw==", + "dev": true, + "dependencies": { + "@types/split2": "^3.2.1", + "@types/tcp-port-used": "^1.0.1", + "@wdio/logger": "^8.1.0", + "fs-extra": "^11.1.0", + "safaridriver": "^0.0.4", + "split2": "^4.1.0", + "tcp-port-used": "^1.0.2" + }, + "engines": { + "node": "^16.13 || >=18" + }, + "peerDependencies": { + "@wdio/types": "^7.0.0 || ^8.0.0-alpha.219", + "webdriverio": "^7.0.0 || ^8.0.0-alpha.219" + }, + "peerDependenciesMeta": { + "@wdio/types": { + "optional": true + }, + "webdriverio": { + "optional": false + } + } + }, "node_modules/webdriver": { "version": "8.5.7", "resolved": "https://registry.npmjs.org/webdriver/-/webdriver-8.5.7.tgz", @@ -7706,12 +7758,27 @@ "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", "dev": true }, + "@types/split2": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/split2/-/split2-3.2.1.tgz", + "integrity": "sha512-7uz3yU+LooBq4yNOzlZD9PU9/1Eu0rTD1MjQ6apOVEoHsPrMUrFw7W8XrvWtesm2vK67SBK9AyJcOXtMpl9bgQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/stack-utils": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", "dev": true }, + "@types/tcp-port-used": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/tcp-port-used/-/tcp-port-used-1.0.1.tgz", + "integrity": "sha512-6pwWTx8oUtWvsiZUCrhrK/53MzKVLnuNSSaZILPy3uMes9QnTrLMar9BDlJArbMOjDcjb3QXFk6Rz8qmmuySZw==", + "dev": true + }, "@types/testing-library__jest-dom": { "version": "5.14.5", "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz", @@ -10985,6 +11052,12 @@ } } }, + "safaridriver": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/safaridriver/-/safaridriver-0.0.4.tgz", + "integrity": "sha512-EgK9Vc2Bk4WaECq1E7ti9GyeQ6JKKQNs40vNOE/b5Ul5dDEt429G0Kj5Nzs4FRS5ZIVa7nBck1TyHuinOYdz2Q==", + "dev": true + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -11383,6 +11456,21 @@ "tcp-port-used": "^1.0.2" } }, + "wdio-safaridriver-service": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wdio-safaridriver-service/-/wdio-safaridriver-service-2.1.0.tgz", + "integrity": "sha512-Zg0aZn8KJo+MwvdOTje2cja4OcrkpoCBhQSzVlhZ3SszMnwhi4VQ1p5TdHB9mWZpuVtsLHWpmx/G7ceJWendVw==", + "dev": true, + "requires": { + "@types/split2": "^3.2.1", + "@types/tcp-port-used": "^1.0.1", + "@wdio/logger": "^8.1.0", + "fs-extra": "^11.1.0", + "safaridriver": "^0.0.4", + "split2": "^4.1.0", + "tcp-port-used": "^1.0.2" + } + }, "webdriver": { "version": "8.5.7", "resolved": "https://registry.npmjs.org/webdriver/-/webdriver-8.5.7.tgz", diff --git a/e2e/package.json b/e2e/package.json index 7eec6b74836..96f9036d3f0 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -41,7 +41,8 @@ "expect": "^29.1.2", "graphql-request": "^5.1.0", "mocha": "^10.2.0", - "wdio-chromedriver-service": "^8.0.1" + "wdio-chromedriver-service": "^8.0.1", + "wdio-safaridriver-service": "^2.1.0" }, "dependencies": { "lit": "^2.4.0", diff --git a/packages/wdio-browser-runner/src/browser/mock.ts b/packages/wdio-browser-runner/src/browser/mock.ts index e0d81bdd524..6ff0ec09f32 100644 --- a/packages/wdio-browser-runner/src/browser/mock.ts +++ b/packages/wdio-browser-runner/src/browser/mock.ts @@ -17,9 +17,4 @@ export const dirname = () => '' export const resolve = () => '' export const sep = '/' export const type = 'browser' - -// mock jest-matcher-utils exports -export const printDiffOrStringify = () => {} -export const printReceived = () => {} -export const printExpected = () => {} export default () => {} diff --git a/packages/wdio-browser-runner/src/vite/constants.ts b/packages/wdio-browser-runner/src/vite/constants.ts index f3a6ebe09f3..7d0553dd6d0 100644 --- a/packages/wdio-browser-runner/src/vite/constants.ts +++ b/packages/wdio-browser-runner/src/vite/constants.ts @@ -2,6 +2,7 @@ import topLevelAwait from 'vite-plugin-top-level-await' import { esbuildCommonjs } from '@originjs/vite-plugin-commonjs' import type { InlineConfig } from 'vite' +import { codeFrameFix } from './plugins/esbuild.js' import type { FrameworkPreset } from '../types.js' export const PRESET_DEPENDENCIES: Record = { @@ -41,7 +42,7 @@ export const DEFAULT_VITE_CONFIG: Partial = { 'expect', 'serialize-error', 'minimatch', 'css-shorthand-properties', 'lodash.merge', 'lodash.zip', 'lodash.clonedeep', 'lodash.pickby', 'lodash.flattendeep', 'aria-query', 'grapheme-splitter', 'css-value', 'rgb2hex', 'p-iteration', 'fast-safe-stringify', - 'deepmerge-ts', 'jest-util' + 'deepmerge-ts', 'jest-util', 'jest-matcher-utils' ], esbuildOptions: { logLevel: 'silent', @@ -51,7 +52,8 @@ export const DEFAULT_VITE_CONFIG: Partial = { }, // Enable esbuild polyfill plugins plugins: [ - esbuildCommonjs(['@testing-library/vue']) + esbuildCommonjs(['@testing-library/vue']), + codeFrameFix() ], }, } diff --git a/packages/wdio-browser-runner/src/vite/plugins/esbuild.ts b/packages/wdio-browser-runner/src/vite/plugins/esbuild.ts new file mode 100644 index 00000000000..5f73ebad7d5 --- /dev/null +++ b/packages/wdio-browser-runner/src/vite/plugins/esbuild.ts @@ -0,0 +1,34 @@ +import fs from 'node:fs/promises' +import type { Plugin, PluginBuild } from 'esbuild' + +export function codeFrameFix () { + return { + name: 'wdio:codeFrameFix', + setup (build: PluginBuild) { + build.onLoad( + { filter: /@babel\/code-frame/, namespace: 'file' }, + + /** + * mock @babel/code-frame as it fails in Safari due + * to usage of chalk + */ + async ({ path: id }: { path: string }) => { + const code = await fs.readFile(id).then( + (buf) => buf.toString(), + () => undefined) + + if (!code) { + return + } + + return { + contents: code.replace('require("@babel/highlight");', /*js*/`{ + shouldHighlight: false, + reset: () => {} + }`) + } + } + ) + } + } +} diff --git a/packages/wdio-browser-runner/src/vite/plugins/testrunner.ts b/packages/wdio-browser-runner/src/vite/plugins/testrunner.ts index 089b5d2f4e7..01c5f0001f4 100644 --- a/packages/wdio-browser-runner/src/vite/plugins/testrunner.ts +++ b/packages/wdio-browser-runner/src/vite/plugins/testrunner.ts @@ -41,8 +41,7 @@ const resolvedVirtualModuleId = '\0' + virtualModuleId * functionality */ const MODULES_TO_MOCK = [ - 'import-meta-resolve', 'puppeteer-core', 'archiver', 'glob', 'devtools', 'ws', - 'jest-matcher-utils', 'decamelize' + 'import-meta-resolve', 'puppeteer-core', 'archiver', 'glob', 'devtools', 'ws', 'decamelize' ] const POLYFILLS = [ diff --git a/packages/wdio-runner/src/browser.ts b/packages/wdio-runner/src/browser.ts index a3422cb8748..79545aa433a 100644 --- a/packages/wdio-runner/src/browser.ts +++ b/packages/wdio-runner/src/browser.ts @@ -141,8 +141,12 @@ export default class BrowserFramework implements Omit { viteError = [{ message: errorElems.map((elem) => elem.innerText).join('\n') }] } } - const loadError = typeof window.__wdioErrors__ === 'undefined' - ? [{ message: `Failed to load test page (title = ${document.title})` }] + const loadError = ( + typeof window.__wdioErrors__ === 'undefined' && + document.title !== 'WebdriverIO Browser Test' && + !document.querySelector('mocha-framework') + ) + ? [{ message: `Failed to load test page (title = "${document.title}", source: ${document.documentElement.innerHTML})` }] : null const errors = viteError || window.__wdioErrors__ || loadError return { failures, errors, hasViteError: Boolean(viteError) }