Plots: A/B screenshot viewer (#2026)

.gitignore

@@ -16,6 +16,7 @@ last-run-results.html
 *.trace.json
 *.devtoolslog.json
 *.screenshots.html
+*.screenshots.json
 *.report.html
 *.report.dom.html
 *.report.json

lighthouse-core/lib/asset-saver.js

@@ -96,7 +96,7 @@ function prepareAssets(artifacts, audits) {
     return chain.then(_ => artifacts.requestScreenshots(trace))
       .then(screenshots => {
         const traceData = Object.assign({}, trace);
-        const html = screenshotDump(screenshots);
+        const screenshotsHTML = screenshotDump(screenshots);
 
         if (audits) {
           const evts = new Metrics(traceData.traceEvents, audits).generateFakeEvents();
@@ -105,7 +105,8 @@ function prepareAssets(artifacts, audits) {
         assets.push({
           traceData,
           devtoolsLog,
-          html
+          screenshotsHTML,
+          screenshots
         });
       });
   }, Promise.resolve())
@@ -130,9 +131,13 @@ function saveAssets(artifacts, audits, pathWithBasename) {
       fs.writeFileSync(devtoolsLogFilename, JSON.stringify(data.devtoolsLog, null, 2));
       log.log('devtools log saved to disk', devtoolsLogFilename);
 
-      const screenshotsFilename = `${pathWithBasename}-${index}.screenshots.html`;
-      fs.writeFileSync(screenshotsFilename, data.html);
-      log.log('screenshots saved to disk', screenshotsFilename);
+      const screenshotsHTMLFilename = `${pathWithBasename}-${index}.screenshots.html`;
+      fs.writeFileSync(screenshotsHTMLFilename, data.screenshotsHTML);
+      log.log('screenshots saved to disk', screenshotsHTMLFilename);
+
+      const screenshotsJSONFilename = `${pathWithBasename}-${index}.screenshots.json`;
+      fs.writeFileSync(screenshotsJSONFilename, JSON.stringify(data.screenshots, null, 2));
+      log.log('screenshots saved to disk', screenshotsJSONFilename);
     });
   });
 }

lighthouse-core/test/lib/asset-saver-test.js

@@ -41,7 +41,7 @@ describe('asset-saver helper', () => {
       requestScreenshots: () => Promise.resolve([]),
     };
     return assetSaver.prepareAssets(artifacts).then(assets => {
-      assert.ok(/<!doctype/gim.test(assets[0].html));
+      assert.ok(/<!doctype/gim.test(assets[0].screenshotsHTML));
     });
   });
 
@@ -74,12 +74,20 @@ describe('asset-saver helper', () => {
       fs.unlinkSync(filename);
     });
 
-    it('screenshots file saved to disk with data', () => {
-      const ssFilename = 'the_file-0.screenshots.html';
-      const ssFileContents = fs.readFileSync(ssFilename, 'utf8');
+    it('screenshots html file saved to disk with data', () => {
+      const ssHTMLFilename = 'the_file-0.screenshots.html';
+      const ssFileContents = fs.readFileSync(ssHTMLFilename, 'utf8');
       assert.ok(/<!doctype/gim.test(ssFileContents));
-      assert.ok(ssFileContents.includes('{"timestamp":674089419.919'));
-      fs.unlinkSync(ssFilename);
+      const expectedScreenshotContent = '{"timestamp":674089419.919';
+      assert.ok(ssFileContents.includes(expectedScreenshotContent), 'unexpected screenshot html');
+      fs.unlinkSync(ssHTMLFilename);
+    });
+
+    it('screenshots json file saved to disk with data', () => {
+      const ssJSONFilename = 'the_file-0.screenshots.json';
+      const ssContents = JSON.parse(fs.readFileSync(ssJSONFilename, 'utf8'));
+      assert.equal(ssContents[0].timestamp, 674089419.919, 'unexpected screenshot json');
+      fs.unlinkSync(ssJSONFilename);
     });
   });
 

plots/README.md

@@ -11,7 +11,7 @@ https://github.com/GoogleChrome/lighthouse/issues/1924
 
 You need to build lighthouse first.
 
-### Commands
+### Generating & viewing charts
 
 ```
 # View all commands
@@ -22,8 +22,9 @@ $ npm run
 $ npm run measure
 
 # Analyze the data to generate a summary file (i.e. out/generatedResults.js)
+# This will launch the charts web page in the browser
 $ npm run analyze
 
-# View visualization
-# Open index.html in browser
-```
+# If you need to view the charts later
+$ npm run open
+```

plots/ab-screenshot/README.md

@@ -0,0 +1,28 @@
+# A/B Screenshot Comparison
+
+This tools enables you to look at how two different versions of perf metrics measure real world sites.
+
+### Generating & viewing charts
+
+```
+# 1. Run measure two times (e.g two different versions of lighthouse)
+
+# In /plots/
+$ node measure.js
+
+# Save the first results into another directory
+$ mv ./out ./out-first
+
+# (e.g. switch versions of lighthouse, modify algorithm)
+$ node measure.js
+
+# Switch to /plots/ab-screenshot
+$ cd ab-screenshot
+
+# Analyze the screenshot data to generate a summary file
+# This will launch the screenshot viewer in the browser
+$ node analyze.js -a ../out-first -b ../out-second
+
+# If you need to open the screenshot viewer later
+$ node open.js
+```

plots/ab-screenshot/analyze.js

@@ -0,0 +1,218 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const opn = require('opn');
+const args = require('yargs').argv;
+
+const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events');
+
+const constants = require('../constants');
+const utils = require('../utils');
+
+/**
+ * Do a comparison analysis of two batches of runs. This helps you compare how a change in
+ * lighthouse can affect perf results in real-world sites.
+ *
+ * Example usage:
+ * node analyze.js -a ./out-first -b ./out-second
+ */
+function main() {
+  const outPathA = args.a;
+  const outPathB = args.b;
+
+  if (!utils.isDir(outPathA) || !utils.isDir(outPathB)) {
+    console.log('ERROR: Make sure both -a and -b point to valid paths'); // eslint-disable-line no-console
+    console.log('a: ', outPathA); // eslint-disable-line no-console
+    console.log('b: ', outPathB); // eslint-disable-line no-console
+    return;
+  }
+
+  const aggregatedScreenshots = {
+    data: aggregate(outPathA, outPathB),
+    a: outPathA,
+    b: outPathB
+  };
+
+  if (!utils.isDir(constants.OUT_PATH)) {
+    fs.mkdirSync(constants.OUT_PATH);
+  }
+  const outFilePath = path.resolve(constants.OUT_PATH, 'screenshotsComparison.js');
+  fs.writeFileSync(
+    outFilePath,
+    `var aggregatedScreenshots = ${JSON.stringify(aggregatedScreenshots, undefined, 2)}`
+  );
+  console.log('Wrote output to:', outFilePath); // eslint-disable-line no-console
+  console.log('Opening the screenshot viewer web page...');  // eslint-disable-line no-console
+  opn(path.resolve(__dirname, 'index.html'));
+}
+
+main();
+
+/**
+ * Aggregates the results from two out paths.
+ * Note: only the first run for each site of each batch is used.
+ * @param {string} outPathA
+ * @param {string} outPathB
+ * @return {!AggregatedScreenshots}
+ */
+function aggregate(outPathA, outPathB) {
+  const results = [];
+
+  fs.readdirSync(outPathA).forEach(siteDir => {
+    const sitePathA = path.resolve(outPathA, siteDir);
+    const sitePathB = path.resolve(outPathB, siteDir);
+
+    // Skip a site if it's not in both batches.
+    if (!utils.isDir(sitePathB)) {
+      return;
+    }
+    const siteScreenshotsComparison = {
+      siteName: siteDir,
+      runA: analyzeSingleRunScreenshots(sitePathA),
+      runB: analyzeSingleRunScreenshots(sitePathB)
+    };
+    results.push(siteScreenshotsComparison);
+  });
+
+  return results;
+}
+
+/**
+ * Analyzes the screenshots for the first run of a particular site.
+ * @param {string} sitePath
+ * @return {!SingleRunScreenshots}
+ */
+function analyzeSingleRunScreenshots(sitePath) {
+  const runDir = sortAndFilterRunFolders(fs.readdirSync(sitePath))[0];
+  const runPath = path.resolve(sitePath, runDir);
+  const lighthouseResultsPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME);
+  const lighthouseResults = JSON.parse(fs.readFileSync(lighthouseResultsPath));
+
+  const fcpTiming = getTiming('ttfcp');
+  const fmpTiming = getTiming('ttfmp');
+  const vc85Timing = getTiming('vc85');
+  const vc100Timing = getTiming('vc100');
+
+  const navStartTimestamp = getTimestamp('navstart');
+
+  const screenshotsPath = path.resolve(runPath, constants.SCREENSHOTS_FILENAME);
+  const screenshots = JSON.parse(fs.readFileSync(screenshotsPath)).map(screenshot => ({
+    timing: Math.round(screenshot.timestamp - navStartTimestamp),
+    datauri: screenshot.datauri
+  }));
+
+  const results = {
+    runName: runPath,
+    screenshots
+  };
+
+  markScreenshots(results, 'isFCP', fcpTiming);
+  markScreenshots(results, 'isFMP', fmpTiming);
+  markScreenshots(results, 'isVC85', vc85Timing);
+  markScreenshots(results, 'isVC100', vc100Timing);
+
+  return results;
+
+  /**
+   * @param {string} id
+   * @return {number}
+   */
+  function getTiming(id) {
+    return Metrics.metricsDefinitions
+      .find(metric => metric.id === id)
+      .getTiming(lighthouseResults.audits);
+  }
+
+  /**
+   * @param {string} id
+   * @return {number}
+   */
+  function getTimestamp(id) {
+    return Metrics.metricsDefinitions
+      .find(metric => metric.id === id)
+      .getTs(lighthouseResults.audits) / 1000; // convert to ms
+  }
+}
+
+/**
+ * @param {!Array<string>} folders
+ * @return {!Array<string>}
+ */
+function sortAndFilterRunFolders(folders) {
+  return folders
+    .filter(folder => folder !== '.DS_Store')
+    .map(folder => Number(folder))
+    .sort((a, b) => a - b)
+    .map(folder => folder.toString());
+}
+
+/**
+ * Marks the first screenshot that happens after a particular perf timing.
+ * @param {SingleRunScreenshots} results
+ * @param {string} key
+ * @param {number} timing
+ */
+function markScreenshots(results, key, timing) {
+  let hasSeenKeyTiming = false;
+  for (const screenshot of results.screenshots) {
+    if (!hasSeenKeyTiming && screenshot.timing > timing) {
+      hasSeenKeyTiming = true;
+      screenshot[key] = true;
+    } else {
+      screenshot[key] = false;
+    }
+  }
+}
+
+/**
+ * @typedef {{
+ *   data: !Array<!SiteScreenshotsComparison>,
+ *   a: string,
+ *   b: string
+ * }}
+ */
+let AggregatedScreenshots; // eslint-disable-line no-unused-vars
+
+/**
+ * @typedef {{
+ *   siteName: string,
+ *   runA: !SingleRunScreenshots,
+ *   runB: !SingleRunScreenshots
+ * }}
+ */
+let SiteScreenshotsComparison; // eslint-disable-line no-unused-vars
+
+/**
+ * @typedef {{runName: string, screenshots: !Array<!Screenshot>}}
+ */
+let SingleRunScreenshots; // eslint-disable-line no-unused-vars
+
+/**
+ * @typedef {{
+ *   datauri: string,
+ *   timing: number,
+ *   isFCP: boolean,
+ *   isFMP: boolean,
+ *   isVC85: boolean,
+ *   isVC100: boolean
+ * }}
+ */
+let Screenshot; // eslint-disable-line no-unused-vars

plots/ab-screenshot/index.html

@@ -0,0 +1,45 @@
+<!--
+Copyright 2017 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<!doctype html>
+<html>
+<head>
+  <link rel="stylesheet" href="screenshot-viewer.css">
+  <title>Screenshots Viewer</title>
+  <link rel="icon" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAADjklEQVR4AWI08P/HQEvAQrxSQKvlECfLFYXx75xCY2qmh89GbNvOMjb3v9jOOlxnFWxj206ebQ3b7q6q+z1rNagu8/zvPSZACAABpeUAA0miMgU7SA7JjCraFGwZwECOwvL75dWjsKgWBKtx0jvWo+vkBAFbACCkByMP6nMn48+AVgXB2fzSCwsv22/lMGlUhmJ0AE7BH8dyUUDbUEgN6RzJRSeaPxhdRYR0Inel+7Hd5lBiFpkMAxACc0394//9C4voFHDiAAGLpuOXebdfdHfctgwJKaZRLRKy6ItrSis6RBnVBgGtbHyKTEmJHQoEXoBCE5BCrDeA2ogMUIGDAKEBDEhUqwgMqBYDjW4DQzmuffVdqff42/ZQYYqVcMXGZsMPyCsH3lyJSetxvEaxAQXdjR1HjfwCdIS7lo2DZke26Qe+MXO12OWkGT0O6oE7vMGkMnkYw4aN1KQgMKExhXqswfiov4+a7MQ11XPnbr/5qpKlgACAAQj94Lu271bN9DUecQasIZlNzG72llRAAKJiAi+/BSHrSFjRvQhg3DEKEqJh08tsmLTx597+f6enr4cc2Zpk57pihfX24dW7RHcOLLUbJYhJSl0ErQCI9BVXH/XrO97QasuvQQSiECa0BrQCIIJp6X9T/r8QG6L71WYSqCoIIGo2BZDUBnS/D9EA9Nun1iYvbM0MFExIDQRoKFatc1Z6zrm5uWeObJotq0BGV9FuQBWq5a4Fw3PPz848rZHstZSuA5FWAFSMP2nOppOOGpl6qh9PCSg0IFyHKjSQyDNQHTru2t75NOEe0fsf246oAmFkI6vCdnWvbQFQFCKx8vCswV8TrDLiDLgH4Nr7RAtNsrC9d8sfk7b8ls4igdNy8CQKAISlsB0FjZfd3Lfp155tf8fKI4BxZZIj/oTdVEAIAcJFOCmzauHG71I7/rdreUAgAqpDP05fDARCAQQARwEIBQSVxq0FyaLvZZtevpHa8WHw8cft6cpxlq8eAJtIhnSbWDf951yx3y13OqUuu5qyGgkxCgGFh9cDihDGbTa6BqvT1lWmrav3bmt2ZMJ4mU6TGgIC4DBzcv/JqAau1WhzSt3x9Ixk/4Jk/8J4ZrrViFMA4W6A7+WK8xcVjvyrOmVD0FbAXokcT48r+xVqLKvuJYbmpNadnlp3mpufJHOe/GXktM+r09bT8kEdq9BRYAbGSgzP7ll82U71Mc+ZFooXgwAAAABJRU5ErkJggg==">
+</head>
+<body>
+  <div id="header">
+    <h1>Screenshots Viewer</h1>
+    <div class="legend">
+        <div id="legend-a">A (top): </div>
+        <div id="legend-b">B (bottom): </div>
+    </div>
+    <div class="settings">
+      <label>
+        Align Timelines:&nbsp
+        <input id="align-control" class="checkbox" type="checkbox" checked="">
+      </label>
+    </div>
+  </div>
+  <div id="container"></div>
+  <div id="image-popover" class="hidden"><img></div>
+
+  <script src="../out/screenshotsComparison.js"></script>
+  <script src="./screenshot-viewer.js"></script>
+</body>
+
+</html>