Plots: make measure script more flexible (CLI args) (#2183)

.gitignore

@@ -47,4 +47,4 @@ lighthouse-cli/types/*.map
 lighthouse-core/report/partials/templates/
 lighthouse-core/report/templates/*.js
 
-plots/out/
+plots/out**

plots/README.md

@@ -14,17 +14,43 @@ You need to build lighthouse first.
 ### Generating & viewing charts
 
 ```
-# View all commands
-$ cd plots
-$ yarn run
-
 # Run lighthouse to collect metrics data
-$ yarn measure
+$ node measure.js
 
 # Analyze the data to generate a summary file (i.e. out/generatedResults.js)
 # This will launch the charts web page in the browser
-$ yarn analyze
+$ node analyze.js
 
 # If you need to view the charts later
-$ yarn open
+$ node open.js
 ```
+
+### Advanced usage
+
+```
+$ node measure.js --help
+
+node measure.js [options]
+
+Lighthouse settings:
+  --disable-device-emulation    Disable Nexus 5X emulation                                                     [boolean]
+  --disable-cpu-throttling      Disable CPU throttling                                                         [boolean]
+  --disable-network-throttling  Disable network throttling                                                     [boolean]
+
+Options to specify sites:
+  --sites-path  Include relative path of a json file with urls to run                              [default: "sites.js"]
+  --subset      Measure a subset of popular sites
+  --site        Include a specific site url to run
+
+Options:
+  --help            Show help                                                                                  [boolean]
+  -n                Number of runs per site                                                                 [default: 3]
+  --reuse-chrome    Reuse the same Chrome instance across all site runs
+  --keep-first-run  If you use --reuse-chrome, by default the first run results are discarded
+
+Examples:
+  node measure.js -n 3 --sites-path ./sample-sites.json
+  node measure.js --site https://google.com/
+  node measure.js --subset
+
+```

plots/ab-screenshot/analyze.js

@@ -20,7 +20,9 @@ const fs = require('fs');
 const path = require('path');
 
 const opn = require('opn');
-const args = require('yargs').argv;
+const args = require('yargs')
+  .default('runs', 1)
+  .argv;
 
 const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events');
 
@@ -84,25 +86,58 @@ function aggregate(outPathA, outPathB) {
     if (!utils.isDir(sitePathB)) {
       return;
     }
-    const siteScreenshotsComparison = {
-      siteName: siteDir,
-      runA: analyzeSingleRunScreenshots(sitePathA),
-      runB: analyzeSingleRunScreenshots(sitePathB)
-    };
-    results.push(siteScreenshotsComparison);
+
+    for (let i = 0; i < args.runs; i++) {
+      const runDirA = getRunDir(sitePathA, i);
+      const runDirB = getRunDir(sitePathB, i);
+
+      const runPathA = path.resolve(sitePathA, runDirA);
+      const runPathB = path.resolve(sitePathB, runDirB);
+
+      const lighthouseFileA = path.resolve(runPathA, constants.LIGHTHOUSE_RESULTS_FILENAME);
+      const lighthouseFileB = path.resolve(runPathB, constants.LIGHTHOUSE_RESULTS_FILENAME);
+
+      if (!utils.isFile(lighthouseFileA) || !utils.isFile(lighthouseFileB)) {
+        continue;
+      }
+
+      const siteScreenshotsComparison = {
+        siteName: `${siteDir} runA: ${runDirA} runB: ${runDirB}`,
+        runA: analyzeSingleRunScreenshots(runPathA),
+        runB: analyzeSingleRunScreenshots(runPathB),
+      };
+      results.push(siteScreenshotsComparison);
+    }
   });
 
   return results;
 }
 
 /**
- * Analyzes the screenshots for the first run of a particular site.
  * @param {string} sitePath
+ * @param {number} runIndex
+ * @return {string}
+ */
+function getRunDir(sitePath, runIndex) {
+  return sortAndFilterRunFolders(fs.readdirSync(sitePath))[runIndex];
+}
+
+/**
+ * @param {!Array<string>} folders
+ * @return {!Array<string>}
+ */
+function sortAndFilterRunFolders(folders) {
+  return folders
+    .filter(folder => folder !== '.DS_Store')
+    .sort((a, b) => Number(a) - Number(b));
+}
+
+/**
+ * Analyzes the screenshots for the first run of a particular site.
+ * @param {string} runPath
  * @return {!SingleRunScreenshots}
  */
-function analyzeSingleRunScreenshots(sitePath) {
-  const runDir = sortAndFilterRunFolders(fs.readdirSync(sitePath))[0];
-  const runPath = path.resolve(sitePath, runDir);
+function analyzeSingleRunScreenshots(runPath) {
   const lighthouseResultsPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME);
   const lighthouseResults = JSON.parse(fs.readFileSync(lighthouseResultsPath));
 
@@ -152,18 +187,6 @@ function analyzeSingleRunScreenshots(sitePath) {
   }
 }
 
-/**
- * @param {!Array<string>} folders
- * @return {!Array<string>}
- */
-function sortAndFilterRunFolders(folders) {
-  return folders
-    .filter(folder => folder !== '.DS_Store')
-    .map(folder => Number(folder))
-    .sort((a, b) => a - b)
-    .map(folder => folder.toString());
-}
-
 /**
  * Marks the first screenshot that happens after a particular perf timing.
  * @param {SingleRunScreenshots} results

plots/bars-per-site.html

@@ -0,0 +1,39 @@
+<!--
+Copyright 2017 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<!doctype html>
+<html>
+
+<head>
+  <script src="https://cdn.plot.ly/plotly-1.25.2.min.js"></script>
+  <style>
+    nav { display: flex; justify-content: center; font-size: 17px; font-family: "Open Sans", verdana, arial, sans-serif; padding: 5px 0 23px; border-bottom: 1px solid black; }
+    nav a { padding: 0 5px; }
+  </style>
+</head>
+
+<body>
+  <nav>
+    View results:
+    <a href="./index.html">grouped by metric</a>
+    <a href="./metrics-per-site.html">grouped by site</a>
+    <a>bar charts per site</a>
+  </nav>
+  <script src="./out/generatedResults.js"></script>
+  <script src="./bars-per-site.js"></script>
+</body>
+
+</html>

plots/bars-per-site.js

@@ -0,0 +1,86 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+/* global Plotly, generatedResults */
+/* eslint-env browser */
+
+const IGNORED_METRICS = new Set(['Navigation Start']);
+
+const metrics = Object.keys(generatedResults).filter(metric => !IGNORED_METRICS.has(metric));
+
+let elementId = 1;
+
+/**
+ * Incrementally renders the plot, otherwise it hangs the browser
+ * because it's generating so many charts.
+ */
+const queuedPlots = [];
+
+function enqueuePlot(fn) {
+  const isFirst = queuedPlots.length == 0;
+  queuedPlots.push(fn);
+  if (isFirst) {
+    renderPlots();
+  }
+}
+
+function renderPlots() {
+  window.requestAnimationFrame(_ => {
+    const plotFn = queuedPlots.shift();
+    if (plotFn) {
+      plotFn();
+      renderPlots();
+    }
+  });
+}
+
+function createChartElement(height = 800) {
+  const div = document.createElement('div');
+  div.style = `width: 100%; height: ${height}px`;
+  div.id = 'chart' + elementId++;
+  document.body.appendChild(div);
+  return div.id;
+}
+
+function generateGroupedBarChart() {
+  const sitesCount = metrics.reduce(
+    (acc, metric) => Math.max(acc, generatedResults[metric].length),
+    0
+  );
+  for (let i = 0; i < sitesCount; i++) {
+    const data = metrics.map(metric => ({
+      y: generatedResults[metric][i].metrics.map(m => m ? m.timing : null),
+      name: metric,
+      type: 'bar'
+    }));
+
+    const layout = {
+      yaxis: {
+        rangemode: 'tozero'
+      },
+      hovermode: 'closest',
+      barmode: 'group',
+      title: generatedResults[metrics[0]][i].site
+    };
+    enqueuePlot(_ => {
+      Plotly.newPlot(createChartElement(), data, layout);
+    });
+  }
+}
+
+generateGroupedBarChart();

plots/index.html

@@ -31,6 +31,7 @@
     View results:
     <a>grouped by metric</a>
     <a href="./metrics-per-site.html">grouped by site</a>
+    <a href="./bars-per-site.html">bar charts per site</a>
   </nav>
 
   <script src="./out/generatedResults.js"></script>