fix: evaluateAsync behavior

lighthouse-cli/test/fixtures/dobetterweb/dbw_tester.html

@@ -293,5 +293,6 @@ <h2>Do better web tester page</h2>
 }
 </script>
 
+<script src="/promise_polyfill.js"></script>
 </body>
 </html>

lighthouse-cli/test/fixtures/static-server.js

@@ -24,7 +24,13 @@ const parseURL = require('url').parse;
 function requestHandler(request, response) {
   const filePath = parseURL(request.url).pathname;
   const queryString = parseURL(request.url).search;
-  const absoluteFilePath = path.join(__dirname, filePath);
+  let absoluteFilePath = path.join(__dirname, filePath);
+  if (filePath === '/promise_polyfill.js') {
+    // evaluateAsync previously had a bug that LH would fail if a page polyfilled Promise.
+    // We bring in a third-party Promise polyfill to ensure we don't still fail.
+    const thirdPartyPath = '../../../lighthouse-core/third_party';
+    absoluteFilePath = path.join(__dirname, `${thirdPartyPath}/promise-polyfill/promise.js`);
+  }
 
   fs.exists(absoluteFilePath, fsExistsCallback);
 

lighthouse-core/gather/driver.js

@@ -129,30 +129,44 @@ class Driver {
   }
 
   /**
-   * Evaluate an expression in the context of the current page. Expression must
-   * evaluate to a Promise. Returns a promise that resolves on asyncExpression's
-   * resolved value.
-   * @param {string} asyncExpression
+   * Evaluate an expression in the context of the current page.
+   * Returns a promise that resolves on the expression's value.
+   * @param {string} expression
    * @return {!Promise<*>}
    */
-  evaluateAsync(asyncExpression) {
+  evaluateAsync(expression) {
     return new Promise((resolve, reject) => {
       // If this gets to 60s and it hasn't been resolved, reject the Promise.
       const asyncTimeout = setTimeout(
         (_ => reject(new Error('The asynchronous expression exceeded the allotted time of 60s'))),
         60000
       );
+
       this.sendCommand('Runtime.evaluate', {
-        expression: asyncExpression,
+        // We need to wrap the raw expression for several purposes
+        // 1. Ensure that the expression will be a native Promise and not a polyfill/non-Promise.
+        // 2. Ensure that errors captured in the Promise are converted into plain-old JS Objects
+        //    so that they can be serialized properly b/c JSON.stringify(new Error('foo')) === '{}'
+        expression: `(function wrapInNativePromise() {
+          const __nativePromise = window.__nativePromise || Promise;
+          return __nativePromise.resolve()
+            .then(_ => ${expression})
+            .catch(${wrapRuntimeEvalErrorInBrowser.toString()});
+        }())`,
         includeCommandLineAPI: true,
         awaitPromise: true,
         returnByValue: true
       }).then(result => {
         clearTimeout(asyncTimeout);
+        const value = result.result.value;
+
         if (result.exceptionDetails) {
-          reject(result.exceptionDetails.exception.value);
+          // An error occurred before we could even create a Promise, should be *very* rare
+          reject(new Error('an unexpected driver error occurred'));
+        } if (value && value.__failedInBrowser) {
+          reject(Object.assign(new Error(), value));
         } else {
-          resolve(result.result.value);
+          resolve(value);
         }
       }).catch(err => {
         clearTimeout(asyncTimeout);
@@ -746,4 +760,23 @@ function captureJSCallUsage(funcRef, set) {
   };
 }
 
+/**
+ * The `exceptionDetails` provided by the debugger protocol does not contain the useful
+ * information such as name, message, and stack trace of the error when it's wrapped in a
+ * promise. Instead, map to a successful object that contains this information.
+ * @param {string|Error} err The error to convert
+ * istanbul ignore next
+ */
+function wrapRuntimeEvalErrorInBrowser(err) {
+  err = err || new Error();
+  const fallbackMessage = typeof err === 'string' ? err : 'unknown error';
+
+  return {
+    __failedInBrowser: true,
+    name: err.name || 'Error',
+    message: err.message || fallbackMessage,
+    stack: err.stack || (new Error()).stack,
+  };
+}
+
 module.exports = Driver;

lighthouse-core/gather/gather-runner.js

@@ -30,8 +30,10 @@ const path = require('path');
  *   C. GatherRunner.setupDriver()
  *     i. assertNoSameOriginServiceWorkerClients
  *     ii. beginEmulation
- *     iii. cleanAndDisableBrowserCaches
- *     iiii. clearDataForOrigin
+ *     iii. enableRuntimeEvents
+ *     iv. evaluateScriptOnLoad rescue native Promise from potential polyfill
+ *     v. cleanAndDisableBrowserCaches
+ *     vi. clearDataForOrigin
  *
  * 2. For each pass in the config:
  *   A. GatherRunner.beforePass()
@@ -90,6 +92,7 @@ class GatherRunner {
     return driver.assertNoSameOriginServiceWorkerClients(options.url)
       .then(_ => driver.beginEmulation(options.flags))
       .then(_ => driver.enableRuntimeEvents())
+      .then(_ => driver.evaluateScriptOnLoad('window.__nativePromise = Promise;'))
       .then(_ => driver.cleanAndDisableBrowserCaches())
       .then(_ => driver.clearDataForOrigin(options.url));
   }

lighthouse-core/gather/gatherers/accessibility.js

@@ -43,9 +43,13 @@ class Accessibility extends Gatherer {
 
   afterPass(options) {
     const driver = options.driver;
+    const expression = `(function () {
+      ${axe};
+      return (${runA11yChecks.toString()}());
+    })()`;
 
     return driver
-        .evaluateAsync(`${axe};(${runA11yChecks.toString()}())`)
+        .evaluateAsync(expression)
         .then(returnedValue => {
           if (!returnedValue) {
             this.artifact = Accessibility._errorAccessibility('Unable to parse axe results');

lighthouse-core/gather/gatherers/dobetterweb/tags-blocking-first-paint.js

@@ -53,7 +53,8 @@ function collectTagsThatBlockFirstPaint() {
         });
       resolve(tagList);
     } catch (e) {
-      reject(`Unable to gather Scripts/Stylesheets/HTML Imports on the page: ${e.message}`);
+      const friendly = 'Unable to gather Scripts/Stylesheets/HTML Imports on the page';
+      reject(new Error(`${friendly}: ${e.message}`));
     }
   });
 }
@@ -117,10 +118,10 @@ class TagsBlockingFirstPaint extends Gatherer {
       .then(artifact => {
         this.artifact = artifact;
       })
-      .catch(debugString => {
+      .catch(err => {
         this.artifact = {
           value: -1,
-          debugString
+          debugString: err.message
         };
       });
   }

lighthouse-core/test/gather/fake-driver.js

@@ -38,6 +38,9 @@ module.exports = {
   enableRuntimeEvents() {
     return Promise.resolve();
   },
+  evaluateScriptOnLoad() {
+    return Promise.resolve();
+  },
   cleanAndDisableBrowserCaches() {},
   clearDataForOrigin() {},
   beginTrace() {

lighthouse-core/test/gather/gatherers/dobetterweb/tags-blocking-first-paint-test.js

@@ -131,7 +131,7 @@ describe('First paint blocking tags', () => {
     return tagsBlockingFirstPaint.afterPass({
       driver: {
         evaluateAsync() {
-          return Promise.reject('such a fail');
+          return Promise.reject(new Error('such a fail'));
         }
       }
     }, traceData).then(_ => {