misc(scripts): more useful lantern debugging output (#5517)

lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js

@@ -10,50 +10,60 @@
 
 const fs = require('fs');
 const path = require('path');
-const execSync = require('child_process').execSync;
 const constants = require('./constants');
+const chalk = require('chalk').default;
 
 const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH;
 const HEAD_PATH = path.resolve(process.cwd(), INPUT_PATH);
 const MASTER_PATH = constants.MASTER_COMPUTED_PATH;
 
-const TMP_DIR = path.join(__dirname, '../../../.tmp');
-const TMP_HEAD_PATH = path.join(TMP_DIR, 'HEAD-for-diff.json');
-const TMP_MASTER_PATH = path.join(TMP_DIR, 'master-for-diff.json');
-
-if (!fs.existsSync(TMP_DIR)) fs.mkdirSync(TMP_DIR);
-
 if (!fs.existsSync(HEAD_PATH) || !fs.existsSync(MASTER_PATH)) {
   throw new Error('Usage $0 <computed file>');
 }
 
-let exitCode = 0;
-
-try {
-  const computedResults = require(HEAD_PATH);
-  const expectedResults = require(MASTER_PATH);
-
-  const sites = [];
-  for (const entry of computedResults.sites) {
-    const lanternValues = entry.lantern;
-    Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key]));
-    sites.push({url: entry.url, ...lanternValues});
-  }
-
-  fs.writeFileSync(TMP_HEAD_PATH, JSON.stringify({sites}, null, 2));
-  fs.writeFileSync(TMP_MASTER_PATH, JSON.stringify(expectedResults, null, 2));
-
-  try {
-    execSync(`git --no-pager diff --color=always --no-index ${TMP_MASTER_PATH} ${TMP_HEAD_PATH}`);
-    console.log('✅  PASS    No changes between expected and computed!');
-  } catch (err) {
-    console.log('❌  FAIL    Changes between expected and computed!\n');
-    console.log(err.stdout.toString());
-    exitCode = 1;
-  }
-} finally {
-  fs.unlinkSync(TMP_HEAD_PATH);
-  fs.unlinkSync(TMP_MASTER_PATH);
+const computedResults = require(HEAD_PATH);
+const expectedResults = require(MASTER_PATH);
+
+/** @type {Array<{url: string, maxDiff: number, diffsForSite: Array<DiffForSite>}>} */
+const diffs = [];
+for (const entry of computedResults.sites) {
+  // @ts-ignore - over-aggressive implicit any on candidate
+  const expectedLantern = expectedResults.sites.find(candidate => entry.url === candidate.url);
+  const actualLantern = entry.lantern;
+
+  let maxDiff = 0;
+  /** @type {DiffForSite[]} */
+  const diffsForSite = [];
+  Object.keys(actualLantern).forEach(metricName => {
+    const actual = Math.round(actualLantern[metricName]);
+    const expected = Math.round(expectedLantern[metricName]);
+    const diff = actual - expected;
+    if (Math.abs(diff) > 0) {
+      maxDiff = Math.max(maxDiff, Math.abs(diff));
+      diffsForSite.push({metricName, actual, expected, diff});
+    }
+  });
+
+  if (maxDiff > 0) diffs.push({url: entry.url, maxDiff, diffsForSite});
+}
+
+if (diffs.length) {
+  console.log(`❌  FAIL    ${diffs.length} change(s) between expected and computed!\n`);
+
+  diffs.sort((a, b) => b.maxDiff - a.maxDiff).forEach(site => {
+    console.log(chalk.magenta(site.url));
+    site.diffsForSite.sort((a, b) => Math.abs(b.diff) - Math.abs(a.diff)).forEach(entry => {
+      const metric = `    - ${entry.metricName.padEnd(25)}`;
+      const diff = entry.diff > 0 ? chalk.yellow(`+${entry.diff}`) : chalk.cyan(`${entry.diff}`);
+      const actual = `${entry.actual} ${chalk.gray('(HEAD)')}`;
+      const expected = `${entry.expected} ${chalk.gray('(master)')}`;
+      console.log(`${metric}${diff}\t${actual}\tvs.\t${expected}`);
+    });
+  });
+
+  process.exit(1);
+} else {
+  console.log('✅  PASS    No changes between expected and computed!');
 }
 
-process.exit(exitCode);
+/** @typedef {{metricName: string, actual: number, expected: number, diff: number}} DiffForSite */

lighthouse-core/scripts/lantern/debug-url.js

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const path = require('path');
+const execFileSync = require('child_process').execFileSync;
+const constants = require('./constants');
+
+const INPUT_URL = process.argv[2];
+if (!INPUT_URL) throw new Error('Usage $0: <url>');
+
+const SITE_INDEX_PATH = path.resolve(process.cwd(), constants.SITE_INDEX_WITH_GOLDEN_PATH);
+const SITE_INDEX_DIR = path.dirname(SITE_INDEX_PATH);
+const RUN_ONCE_PATH = path.join(__dirname, 'run-once.js');
+
+const siteIndex = require(SITE_INDEX_PATH);
+// @ts-ignore - over-aggressive implicit any on site
+const site = siteIndex.sites.find(site => site.url === INPUT_URL);
+if (!site) throw new Error(`Could not find with site URL ${INPUT_URL}`);
+
+const trace = path.join(SITE_INDEX_DIR, site.unthrottled.tracePath);
+const log = path.join(SITE_INDEX_DIR, site.unthrottled.devtoolsLogPath);
+process.env.LANTERN_DEBUG = 'true';
+execFileSync('node', ['--inspect-brk', RUN_ONCE_PATH, trace, log], {stdio: 'inherit'});

lighthouse-core/scripts/lantern/run-once.js

@@ -6,21 +6,33 @@
  */
 'use strict';
 
+const fs = require('fs');
 const path = require('path');
-const LH_ROOT_DIR = path.join(__dirname, '../../../');
+const PredictivePerf = require('../../../lighthouse-core/audits/predictive-perf');
+const Runner = require('../../../lighthouse-core/runner');
+const Simulator = require('../../../lighthouse-core/lib/dependency-graph/simulator/simulator');
+const traceSaver = require('../../../lighthouse-core/lib/lantern-trace-saver');
 
 if (process.argv.length !== 4) throw new Error('Usage $0 <trace file> <devtools file>');
 
 async function run() {
-  const PredictivePerf = require(path.join(LH_ROOT_DIR, 'lighthouse-core/audits/predictive-perf'));
-  const Runner = require(path.join(LH_ROOT_DIR, 'lighthouse-core/runner'));
-
-  const traces = {defaultPass: require(process.argv[2])};
+  const tracePath = require.resolve(process.argv[2]);
+  const traces = {defaultPass: require(tracePath)};
   const devtoolsLogs = {defaultPass: require(process.argv[3])};
   const artifacts = {traces, devtoolsLogs, ...Runner.instantiateComputedArtifacts()};
 
+  // @ts-ignore - We don't need the full artifacts
   const result = await PredictivePerf.audit(artifacts);
   process.stdout.write(JSON.stringify(result.details.items[0], null, 2));
+
+  // Dump the TTI graph with simulated timings to a trace if LANTERN_DEBUG is enabled
+  const pessimisticTTINodeTimings = Simulator.ALL_NODE_TIMINGS.get('pessimisticInteractive');
+  if (process.env.LANTERN_DEBUG && pessimisticTTINodeTimings) {
+    const outputTraceFile = path.basename(tracePath).replace(/.trace.json$/, '.lantern.trace.json');
+    const outputTracePath = path.join(__dirname, '../../../.tmp', outputTraceFile);
+    const trace = traceSaver.convertNodeTimingsToTrace(pessimisticTTINodeTimings);
+    fs.writeFileSync(outputTracePath, JSON.stringify(trace, null, 2));
+  }
 }
 
 run().catch(err => {

package.json

@@ -81,6 +81,7 @@
     "@types/yargs": "^8.0.2",
     "babel-core": "^6.26.0",
     "bundlesize": "^0.14.4",
+    "chalk": "^2.4.1",
     "codecov": "^2.2.0",
     "commitizen": "^2.9.6",
     "conventional-changelog-cli": "^1.3.4",

yarn.lock

@@ -303,6 +303,12 @@ ansi-styles@^3.1.0:
   dependencies:
     color-convert "^1.9.0"
 
+ansi-styles@^3.2.1:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d"
+  dependencies:
+    color-convert "^1.9.0"
+
 append-transform@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/append-transform/-/append-transform-0.4.0.tgz#d76ebf8ca94d276e247a36bad44a4b74ab611991"
@@ -843,6 +849,14 @@ chalk@2.1.0, chalk@^2.0.0, chalk@^2.1.0:
     escape-string-regexp "^1.0.5"
     supports-color "^4.0.0"
 
+chalk@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.1.tgz#18c49ab16a037b6eb0152cc83e3471338215b66e"
+  dependencies:
+    ansi-styles "^3.2.1"
+    escape-string-regexp "^1.0.5"
+    supports-color "^5.3.0"
+
 chrome-devtools-frontend@1.0.401423:
   version "1.0.401423"
   resolved "https://registry.yarnpkg.com/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.401423.tgz#32a89b8d04e378a494be3c8d63271703be1c04ea"
@@ -2377,6 +2391,10 @@ has-flag@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51"
 
+has-flag@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd"
+
 has-gulplog@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/has-gulplog/-/has-gulplog-0.1.0.tgz#6414c82913697da51590397dafb12f22967811ce"
@@ -4795,6 +4813,12 @@ supports-color@^4.0.0:
   dependencies:
     has-flag "^2.0.0"
 
+supports-color@^5.3.0:
+  version "5.4.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54"
+  dependencies:
+    has-flag "^3.0.0"
+
 symbol-tree@^3.2.1:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.2.tgz#ae27db38f660a7ae2e1c3b7d1bc290819b8519e6"