test: replace Karma with Webdriver.IO (#17126)

* test: replace Karma with Webdriver.IO

The current test framework for browser testing (Karma) is not maintained anymore and WebdriverIO provides a more modern stack that allows to test in different browser.

This patch replaces these test frameworks.

fixes: #17009

* update webdriverio deps

* PR feedback

* adjust tests

* build eslint before running tests

* make test file an esm file

* revert more esm changes

* make it work

* remove return value

* custom log dir for wdio tests

* auto detect chromedriver

* bump timeout, store logs

* bump timeout again

* update wdio deps

* update wdio deps

* set log level to trace

* update wdio deps and unskip tests

* no need to have this be an async test

* update deps

* make path spec file explicit

* remove Chromedriver deps

* removed wdio command

.github/workflows/ci.yml

@@ -75,6 +75,12 @@ jobs:
     - name: Install Packages
       run: npm install
     - name: Test
-      run: node Makefile karma
+      run: node Makefile wdio
     - name: Fuzz Test
       run: node Makefile fuzz
+    - uses: actions/upload-artifact@v3
+      if: failure()
+      with:
+        name: logs
+        path: |
+          wdio-logs/*.log

.gitignore

@@ -3,6 +3,8 @@
 test.js
 coverage/
 build/
+logs
+wdio-logs
 npm-debug.log
 yarn-error.log
 .pnpm-debug.log

Makefile.js

@@ -628,12 +628,10 @@ target.mocha = () => {
     }
 };
 
-target.karma = () => {
+target.wdio = () => {
     echo("Running unit tests on browsers");
-
     target.webpack("production");
-
-    const lastReturn = exec(`${getBinFile("karma")} start karma.conf.js`);
+    const lastReturn = exec(`${getBinFile("wdio")} run wdio.conf.js`);
 
     if (lastReturn.code !== 0) {
         exit(1);
@@ -643,7 +641,7 @@ target.karma = () => {
 target.test = function() {
     target.checkRuleFiles();
     target.mocha();
-    target.karma();
+    target.wdio();
     target.fuzz({ amount: 150, fuzzBrokenAutofixes: false });
     target.checkLicenses();
 };

lib/config/rule-validator.js

@@ -9,7 +9,8 @@
 // Requirements
 //-----------------------------------------------------------------------------
 
-const ajv = require("../shared/ajv")();
+const ajvImport = require("../shared/ajv");
+const ajv = ajvImport();
 const {
     parseRuleId,
     getRuleFromConfig,

package.json

@@ -101,6 +101,11 @@
   "devDependencies": {
     "@babel/core": "^7.4.3",
     "@babel/preset-env": "^7.4.3",
+    "@wdio/browser-runner": "^8.14.6",
+    "@wdio/cli": "^8.14.6",
+    "@wdio/concise-reporter": "^8.14.0",
+    "@wdio/globals": "^8.14.6",
+    "@wdio/mocha-framework": "^8.14.0",
     "babel-loader": "^8.0.5",
     "c8": "^7.12.0",
     "chai": "^4.0.1",
@@ -124,11 +129,6 @@
     "glob": "^7.1.6",
     "got": "^11.8.3",
     "gray-matter": "^4.0.3",
-    "karma": "^6.1.1",
-    "karma-chrome-launcher": "^3.1.0",
-    "karma-mocha": "^2.0.1",
-    "karma-mocha-reporter": "^2.2.5",
-    "karma-webpack": "^5.0.0",
     "lint-staged": "^11.0.0",
     "load-perf": "^0.2.0",
     "markdownlint": "^0.25.1",
@@ -148,12 +148,14 @@
     "pirates": "^4.0.5",
     "progress": "^2.0.3",
     "proxyquire": "^2.0.1",
-    "puppeteer": "^13.7.0",
     "recast": "^0.20.4",
     "regenerator-runtime": "^0.13.2",
+    "rollup-plugin-node-polyfills": "^0.2.1",
     "semver": "^7.5.3",
     "shelljs": "^0.8.2",
     "sinon": "^11.0.0",
+    "vite-plugin-commonjs": "^0.8.2",
+    "webdriverio": "^8.14.6",
     "webpack": "^5.23.0",
     "webpack-cli": "^4.5.0",
     "yorkie": "^2.0.0"

tests/lib/linter/linter.js

@@ -9,7 +9,7 @@
 // Requirements
 //------------------------------------------------------------------------------
 
-const assert = require("chai").assert,
+const { assert } = require("chai"),
     sinon = require("sinon"),
     espree = require("espree"),
     esprima = require("esprima"),
@@ -7263,12 +7263,12 @@ var a = "test2";
 
         it("should have file path passed to it", () => {
             const code = "/* this is code */";
-            const parseSpy = sinon.spy(testParsers.stubParser, "parse");
+            const parseSpy = { parse: sinon.spy() };
 
-            linter.defineParser("stub-parser", testParsers.stubParser);
+            linter.defineParser("stub-parser", parseSpy);
             linter.verify(code, { parser: "stub-parser" }, filename, true);
 
-            sinon.assert.calledWithMatch(parseSpy, "", { filePath: filename });
+            sinon.assert.calledWithMatch(parseSpy.parse, "", { filePath: filename });
         });
 
         it("should not report an error when JSX code contains a spread operator and JSX is enabled", () => {
@@ -8068,16 +8068,16 @@ describe("Linter with FlatConfigArray", () => {
 
                     it("should have file path passed to it", () => {
                         const code = "/* this is code */";
-                        const parseSpy = sinon.spy(testParsers.stubParser, "parse");
+                        const parseSpy = { parse: sinon.spy() };
                         const config = {
                             languageOptions: {
-                                parser: testParsers.stubParser
+                                parser: parseSpy
                             }
                         };
 
                         linter.verify(code, config, filename, true);
 
-                        sinon.assert.calledWithMatch(parseSpy, "", { filePath: filename });
+                        sinon.assert.calledWithMatch(parseSpy.parse, "", { filePath: filename });
                     });
 
                     it("should not report an error when JSX code contains a spread operator and JSX is enabled", () => {