test-browser: use native event handling (#8823)

alxndrsn · web-flow · commit dc97aedf449a · 2023-11-22T12:00:58.000+03:00
Previously the browser test integration with Selenium was written to poll the
browser for logs and test events at a fixed interval.  With playwright, we can
hook directly into the browser events, and handle them immediately.

This reduces lag for general logging, result reporting, and test completion.
diff --git a/bin/test-browser.js b/bin/test-browser.js
@@ -57,10 +57,10 @@ testUrl += '?';
 testUrl += new URLSearchParams(pickBy(qs, identity));
 
 class RemoteRunner {
-  constructor() {
+  constructor(browser) {
+    this.browser = browser;
     this.handlers = {};
-    this.completed = false;
-    this.failed = false;
+    this.handleEvent = this.handleEvent.bind(this);
   }
 
   on(name, handler) {
@@ -72,46 +72,47 @@ class RemoteRunner {
     handlers[name].push(handler);
   }
 
-  handleEvents(events) {
-    var handlers = this.handlers;
-
-    events.forEach((event) => {
-      this.completed = this.completed || event.name === 'end';
-      this.failed = this.failed || event.name === 'fail';
-
+  async handleEvent(event) {
+    try {
       var additionalProps = ['pass', 'fail', 'pending'].indexOf(event.name) === -1 ? {} : {
         slow: event.obj.slow ? function () { return event.obj.slow; } : function () { return 60; },
         fullTitle: event.obj.fullTitle ? function () { return event.obj.fullTitle; } : undefined
       };
       var obj = Object.assign({}, event.obj, additionalProps);
 
-      handlers[event.name].forEach(function (handler) {
+      this.handlers[event.name].forEach(function (handler) {
         handler(obj, event.err);
       });
 
-      if (event.logs && event.logs.length > 0) {
-        event.logs.forEach(function (line) {
-          if (line.type === 'log') {
-            console.log(line.content);
-          } else if (line.type === 'error') {
-            console.error(line.content);
-          } else {
-            console.error('Invalid log line', line);
-          }
-        });
-        console.log();
+      switch (event.name) {
+        case 'fail': this.handleFailed(); break;
+        case 'end': this.handleEnd(); break;
       }
-    });
-  }
+    } catch (e) {
+      console.error('Tests failed:', e);
 
-  bail() {
-    var handlers = this.handlers;
+      await this.browser.close();
+      process.exit(3);
+    }
+  }
 
-    handlers['end'].forEach(function (handler) {
-      handler();
-    });
+  async handleEnd(failed) {
+    await this.browser.close();
+    process.exit(!process.env.PERF && failed ? 1 : 0);
+  }
 
-    this.completed = true;
+  handleFailed() {
+    if (bail) {
+      try {
+        this.handlers['end'].forEach(function (handler) {
+          handler();
+        });
+      } catch (e) {
+        console.log('An error occurred while bailing:', e);
+      } finally {
+        this.handleEnd(true);
+      }
+    }
   }
 }
 
@@ -142,7 +143,13 @@ async function startTest() {
 
   console.log('Starting', browserName, 'on', testUrl);
 
-  const runner = new RemoteRunner();
+  const options = {
+    headless: true,
+  };
+  const browser = await playwright[browserName].launch(options);
+  const page = await browser.newPage();
+
+  const runner = new RemoteRunner(browser);
   new MochaSpecReporter(runner);
   new BenchmarkConsoleReporter(runner);
 
@@ -154,11 +161,14 @@ async function startTest() {
     new BenchmarkJsonReporter(runner);
   }
 
-  const options = {
-    headless: true,
-  };
-  const browser = await playwright[browserName].launch(options);
-  const page = await browser.newPage();
+  page.exposeFunction('handleMochaEvent', runner.handleEvent);
+  page.addInitScript(() => {
+    window.addEventListener('message', (e) => {
+      if (e.data.type === 'mocha') {
+        window.handleMochaEvent(e.data.details);
+      }
+    });
+  });
 
   page.on('pageerror', err => {
     if (browserName === 'webkit' && err.toString()
@@ -174,46 +184,14 @@ async function startTest() {
     process.exit(1);
   });
 
-  if (process.env.BROWSER_CONSOLE) {
-    page.on('console', message => {
-      const { url, lineNumber } = message.location();
-      console.log('BROWSER', message.type().toUpperCase(), `${url}:${lineNumber}`, message.text());
-    });
-  }
+  page.on('console', message => {
+    console.log(message.text());
+  });
 
   await page.goto(testUrl);
 
   const userAgent = await page.evaluate('navigator.userAgent');
   console.log('Testing on:', userAgent);
-
-  const interval = setInterval(async () => {
-    try {
-      const events = await page.evaluate('window.testEvents()');
-      runner.handleEvents(events);
-
-      if (runner.completed || (runner.failed && bail)) {
-        if (!runner.completed && runner.failed) {
-          try {
-            runner.bail();
-          } catch (e) {
-            // Temporary debugging of bailing failure
-            console.log('An error occurred while bailing:');
-            console.log(e);
-          }
-        }
-
-        clearInterval(interval);
-        await browser.close();
-        process.exit(!process.env.PERF && runner.failed ? 1 : 0);
-      }
-    } catch (e) {
-      console.error('Tests failed:', e);
-
-      clearInterval(interval);
-      await browser.close();
-      process.exit(3);
-    }
-  }, 1000);
 }
 
 devserver.start(function () {
diff --git a/tests/integration/webrunner.js b/tests/integration/webrunner.js
@@ -8,72 +8,31 @@
     window.removeEventListener("load", startTests);
 
     if (remote) {
-      // Capture logs for test runner output
-      var logs = [];
-
-      (function () {
-
-        function serializeLogItem(obj, filter, space) {
-          if (typeof obj === 'string') {
-            return obj;
-          } else if (obj instanceof Error) {
-            return obj.stack;
-          } else {
-            return JSON.stringify(obj, filter, space);
-          }
-        }
-
-        function wrappedLog(oldLog, type) {
-          return function () {
-            var args = Array.prototype.slice.call(arguments);
-            logs.push({
-              type,
-              content: args.map(function (arg) {
-                return serializeLogItem(arg);
-              }).join(' ')
-            });
-            oldLog.apply(console, arguments);
-          };
-        }
-
-        console.log = wrappedLog(console.log, 'log');
-        console.error = wrappedLog(console.error, 'error');
-
-      })();
-
-      // Capture test events for test runner output
-      var testEventsBuffer = [];
-
-      window.testEvents = function () {
-        var events = testEventsBuffer;
-        testEventsBuffer = [];
-        return events;
-      };
-
       mocha.reporter(function (runner) {
         var eventNames = ['start', 'end', 'suite', 'suite end', 'pass', 'pending', 'fail'];
         eventNames.forEach(function (name) {
           runner.on(name, function (obj, err) {
-            testEventsBuffer.push({
-              name,
-              obj: obj && {
-                root: obj.root,
-                title: obj.title,
-                duration: obj.duration,
-                slow: typeof obj.slow === 'function' ? obj.slow() : undefined,
-                fullTitle: typeof obj.fullTitle === 'function' ? obj.fullTitle() : undefined
-              },
-              err: err && {
-                actual: err.actual,
-                expected: err.expected,
-                showDiff: err.showDiff,
-                message: err.message,
-                stack: err.stack,
-                uncaught: err.uncaught
+            window.postMessage({
+              type: 'mocha',
+              details: {
+                name,
+                obj: obj && {
+                  root: obj.root,
+                  title: obj.title,
+                  duration: obj.duration,
+                  slow: typeof obj.slow === 'function' ? obj.slow() : undefined,
+                  fullTitle: typeof obj.fullTitle === 'function' ? obj.fullTitle() : undefined
+                },
+                err: err && {
+                  actual: err.actual,
+                  expected: err.expected,
+                  showDiff: err.showDiff,
+                  message: err.message,
+                  stack: err.stack,
+                  uncaught: err.uncaught
+                },
               },
-              logs
             });
-            logs = [];
           });
         });
       });
diff --git a/tests/performance/perf.reporter.js b/tests/performance/perf.reporter.js
@@ -9,14 +9,12 @@ var results = {
   tests: {}
 };
 
-// Capture test events for test runner output
-var testEventsBuffer = [];
-
-global.testEvents = function () {
-  var events = testEventsBuffer;
-  testEventsBuffer = [];
-  return events;
-};
+function emitMochaEvent(details) {
+  // NodeJS perf testing just reports with console.log().
+  if (typeof window !== 'undefined') {
+    window.postMessage({ type: 'mocha', details });
+  }
+}
 
 // fix for Firefox max timing entries capped to 150:
 // https://bugzilla.mozilla.org/show_bug.cgi?id=1331135
@@ -38,11 +36,11 @@ exports.log = log;
 
 exports.startSuite = function (suiteName) {
   log('Starting suite: ' + suiteName + '\n\n');
-  testEventsBuffer.push({ name: 'suite', obj: { title: suiteName } });
+  emitMochaEvent({ name: 'suite', obj: { title: suiteName } });
 };
 
 exports.endSuite = function (suiteName) {
-  testEventsBuffer.push({ name: 'suite end', obj: { title: suiteName } });
+  emitMochaEvent({ name: 'suite end', obj: { title: suiteName } });
 };
 
 exports.start = function (testCase, iter) {
@@ -61,8 +59,8 @@ exports.end = function (testCase) {
   obj.numIterations = obj.iterations.length;
   delete obj.iterations; // keep it simple when reporting
   log('median: ' + obj.median + ' ms\n');
-  testEventsBuffer.push({ name: 'pass', obj: { title: testCase.name } });
-  testEventsBuffer.push({ name: 'benchmark:result', obj });
+  emitMochaEvent({ name: 'pass', obj: { title: testCase.name } });
+  emitMochaEvent({ name: 'benchmark:result', obj });
 };
 
 exports.startIteration = function (testCase) {
@@ -75,7 +73,7 @@ exports.endIteration = function (testCase) {
 };
 
 exports.startAll = function () {
-  testEventsBuffer.push({ name: 'start' });
+  emitMochaEvent({ name: 'start' });
 };
 
 exports.complete = function (adapter) {
@@ -97,6 +95,6 @@ exports.complete = function (adapter) {
   results.adapter = adapter;
   console.log('=>', JSON.stringify(results, null, '  '), '<=');
   log('\nTests Complete!\n\n');
-  testEventsBuffer.push({ name: 'end', obj: results });
+  emitMochaEvent({ name: 'end', obj: results });
 };
 
