tests(lantern): add lantern regression test scripts

.gitignore

@@ -2,6 +2,7 @@ node_modules
 .DS_Store
 npm-debug.log
 .vscode
+.tmp
 
 lighthouse-extension/dist
 lighthouse-core/third_party/src

.travis.yml

@@ -13,6 +13,7 @@ cache:
   yarn: true
   directories:
     - node_modules
+    - lantern-data
     - lighthouse-extension/node_modules
     - lighthouse-viewer/node_modules
     - /home/travis/.rvm/gems/
@@ -39,6 +40,7 @@ script:
   - yarn smoke:silentcoverage
   - yarn test-extension
   - yarn test-viewer
+  - yarn test-lantern
   # _JAVA_OPTIONS is breaking parsing of compiler output. See #3338.
   - unset _JAVA_OPTIONS
   # FIXME(paulirish): re-enable after a roll of LH->CDT

lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+const execSync = require('child_process').execSync;
+const constants = require('./constants');
+
+const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH;
+const HEAD_PATH = path.resolve(process.cwd(), INPUT_PATH);
+const MASTER_PATH = constants.MASTER_COMPUTED_PATH;
+
+const TMP_DIR = path.join(__dirname, '../../../.tmp');
+const TMP_HEAD_PATH = path.join(TMP_DIR, 'HEAD-for-diff.json');
+const TMP_MASTER_PATH = path.join(TMP_DIR, 'master-for-diff.json');
+
+if (!fs.existsSync(TMP_DIR)) fs.mkdirSync(TMP_DIR);
+
+if (!fs.existsSync(HEAD_PATH) || !fs.existsSync(MASTER_PATH)) {
+  throw new Error('Usage $0 <computed file>');
+}
+
+let exitCode = 0;
+
+try {
+  const computedResults = require(HEAD_PATH);
+  const expectedResults = require(MASTER_PATH);
+
+  const sites = [];
+  for (const entry of computedResults.sites) {
+    const lanternValues = entry.lantern;
+    Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key]));
+    sites.push({url: entry.url, ...lanternValues});
+  }
+
+  fs.writeFileSync(TMP_HEAD_PATH, JSON.stringify({sites}, null, 2));
+  fs.writeFileSync(TMP_MASTER_PATH, JSON.stringify(expectedResults, null, 2));
+
+  try {
+    execSync(`git --no-pager diff --color=always --no-index ${TMP_MASTER_PATH} ${TMP_HEAD_PATH}`);
+    console.log('✅  PASS    No changes between expected and computed!');
+  } catch (err) {
+    console.log('❌  FAIL    Changes between expected and computed!\n');
+    console.log(err.stdout.toString());
+    exitCode = 1;
+  }
+} finally {
+  fs.unlinkSync(TMP_HEAD_PATH);
+  fs.unlinkSync(TMP_MASTER_PATH);
+}
+
+process.exit(exitCode);

lighthouse-core/scripts/lantern/constants.js

@@ -0,0 +1,16 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const path = require('path');
+
+/* eslint-disable max-len */
+
+module.exports = {
+  SITE_INDEX_WITH_GOLDEN_PATH: './lantern-data/site-index-plus-golden-expectations.json',
+  SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH: path.join(__dirname, '../../../.tmp/site-index-plus-golden-expectations-plus-computed.json'),
+  MASTER_COMPUTED_PATH: path.join(__dirname, '../../test/fixtures/lantern-master-computed-values.json'),
+};

lighthouse-core/scripts/lantern/download-traces.sh

@@ -1,13 +1,22 @@
 #!/bin/bash
 
+set -e
+
 DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 LH_ROOT_PATH="$DIRNAME/../../.."
 cd $LH_ROOT_PATH
 
+if [[ -f lantern-data/site-index-plus-golden-expectations.json ]] && ! [[ "$FORCE" ]]; then
+  echo "Lantern data already detected, done."
+  exit 0
+fi
+
+rm -rf lantern-data/
+mkdir lantern-data/ && cd lantern-data/
+
 # snapshot of ~100 traces with no throttling recorded 2017-12-06 on a HP z840 workstation
 TAR_URL="https://drive.google.com/a/chromium.org/uc?id=1_w2g6fQVLgHI62FApsyUDejZyHNXMLm0&export=download"
 curl -o lantern-traces.tar.gz -L $TAR_URL
 
 tar -xzf lantern-traces.tar.gz
-mv lantern-traces-subset lantern-data
 rm lantern-traces.tar.gz

lighthouse-core/scripts/lantern/evaluate-results.js → lighthouse-core/scripts/lantern/print-correlations.js

@@ -8,16 +8,20 @@
 
 /* eslint-disable no-console */
 
+const fs = require('fs');
 const path = require('path');
+const constants = require('./constants');
 
 const GOOD_ABSOLUTE_THRESHOLD = 0.2;
 const OK_ABSOLUTE_THRESHOLD = 0.5;
 
 const GOOD_RANK_THRESHOLD = 0.1;
 
-if (!process.argv[2]) throw new Error('Usage $0 <computed summary file>');
+const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH;
+const COMPUTATIONS_PATH = path.resolve(process.cwd(), INPUT_PATH);
+
+if (!fs.existsSync(COMPUTATIONS_PATH)) throw new Error('Usage $0 <computed summary file>');
 
-const COMPUTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]);
 /** @type {{sites: LanternSiteDefinition[]}} */
 const expectations = require(COMPUTATIONS_PATH);
 
@@ -35,7 +39,7 @@ const totalOk = [];
 const totalBad = [];
 
 /**
- * @param {keyof LanternSiteDefinition} metric
+ * @param {keyof TargetMetrics} metric
  * @param {keyof LanternMetrics} lanternMetric
  */
 function evaluateBuckets(metric, lanternMetric) {
@@ -52,8 +56,7 @@ function evaluateBuckets(metric, lanternMetric) {
   const rankErrors = [];
   const percentErrors = [];
   for (const entry of entries) {
-    // @ts-ignore
-    const expected = Math.round(entry[metric]);
+    const expected = Math.round(entry.wpt3g[metric]);
     if (expected === 0) continue;
 
     const expectedRank = sortedByMetric.indexOf(entry);
@@ -122,9 +125,9 @@ evaluateBuckets('speedIndex', 'roughEstimateOfSI');
 
 const total = totalGood.length + totalOk.length + totalBad.length;
 console.log('\n----    Summary Stats    ----');
-console.log(`Good: ${Math.round(totalGood.length / total * 100)}%`);
-console.log(`OK: ${Math.round(totalOk.length / total * 100)}%`);
-console.log(`Bad: ${Math.round(totalBad.length / total * 100)}%`);
+console.log(`Good: ${Math.round((totalGood.length / total) * 100)}%`);
+console.log(`OK: ${Math.round((totalOk.length / total) * 100)}%`);
+console.log(`Bad: ${Math.round((totalBad.length / total) * 100)}%`);
 
 console.log('\n----    Worst10 Sites    ----');
 for (const entry of totalBad.sort((a, b) => b.rankDiff - a.rankDiff).slice(0, 10)) {
@@ -141,14 +144,8 @@ for (const entry of totalBad.sort((a, b) => b.rankDiff - a.rankDiff).slice(0, 10
 /**
  * @typedef LanternSiteDefinition
  * @property {string} url
- * @property {string} tracePath
- * @property {string} devtoolsLogPath
+ * @property {TargetMetrics} wpt3g
  * @property {LanternMetrics} lantern
- * @property {number} [firstContentfulPaint]
- * @property {number} [firstMeaningfulPaint]
- * @property {number} [timeToFirstInteractive]
- * @property {number} [timeToConsistentlyInteractive]
- * @property {number} [speedIndex]
  */
 
 /**
@@ -162,6 +159,15 @@ for (const entry of totalBad.sort((a, b) => b.rankDiff - a.rankDiff).slice(0, 10
  * @property {number} rankDiffAsPercent
  */
 
+/**
+ * @typedef TargetMetrics
+ * @property {number} [firstContentfulPaint]
+ * @property {number} [firstMeaningfulPaint]
+ * @property {number} [timeToFirstInteractive]
+ * @property {number} [timeToConsistentlyInteractive]
+ * @property {number} [speedIndex]
+ */
+
 /**
  * @typedef LanternMetrics
  * @property {number} optimisticFCP
@@ -179,4 +185,4 @@ for (const entry of totalBad.sort((a, b) => b.rankDiff - a.rankDiff).slice(0, 10
  * @property {number} roughEstimateOfSI
  * @property {number} roughEstimateOfTTFCPUI
  * @property {number} roughEstimateOfTTI
-  */
+ */

lighthouse-core/scripts/lantern/run-all-expectations.js → lighthouse-core/scripts/lantern/run-on-all-assets.js

@@ -11,17 +11,20 @@
 const fs = require('fs');
 const path = require('path');
 const execFileSync = require('child_process').execFileSync;
+const constants = require('./constants');
 
-if (!process.argv[2]) throw new Error('Usage $0 <expectations file>');
-
+const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_PATH;
+const SITE_INDEX_PATH = path.resolve(process.cwd(), INPUT_PATH);
+const SITE_INDEX_DIR = path.dirname(SITE_INDEX_PATH);
 const RUN_ONCE_PATH = path.join(__dirname, 'run-once.js');
-const EXPECTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]);
-const EXPECTATIONS_DIR = path.dirname(EXPECTATIONS_PATH);
-const expectations = require(EXPECTATIONS_PATH);
+
+if (!fs.existsSync(SITE_INDEX_PATH)) throw new Error('Usage $0 <expectations file>');
+
+const expectations = require(SITE_INDEX_PATH);
 
 for (const site of expectations.sites) {
-  const trace = path.join(EXPECTATIONS_DIR, site.tracePath);
-  const log = path.join(EXPECTATIONS_DIR, site.devtoolsLogPath);
+  const trace = path.join(SITE_INDEX_DIR, site.unthrottled.tracePath);
+  const log = path.join(SITE_INDEX_DIR, site.unthrottled.devtoolsLogPath);
 
   console.log('Running', site.url, '...');
   const rawOutput = execFileSync(RUN_ONCE_PATH, [trace, log])
@@ -33,5 +36,5 @@ for (const site of expectations.sites) {
   Object.assign(site, {lantern});
 }
 
-const computedSummaryPath = path.join(EXPECTATIONS_DIR, 'lantern-computed.json');
-fs.writeFileSync(computedSummaryPath, JSON.stringify(expectations, null, 2));
+// eslint-disable-next-line max-len
+fs.writeFileSync(constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH, JSON.stringify(expectations, null, 2));

lighthouse-core/scripts/lantern/update-master-lantern-values.js

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+const execFileSync = require('child_process').execFileSync;
+const prettyJSONStringify = require('pretty-json-stringify');
+const constants = require('./constants');
+
+const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_PATH;
+const SITE_INDEX_PATH = path.resolve(process.cwd(), INPUT_PATH);
+const HEAD_COMPUTED_PATH = constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH;
+const RUN_ALL_SCRIPT_PATH = path.join(__dirname, 'run-on-all-assets.js');
+const OUTPUT_PATH = constants.MASTER_COMPUTED_PATH;
+
+if (!fs.existsSync(HEAD_COMPUTED_PATH) || process.env.FORCE) {
+  if (!fs.existsSync(SITE_INDEX_PATH)) throw new Error('Usage $0 <expectations file>');
+  execFileSync(RUN_ALL_SCRIPT_PATH, [SITE_INDEX_PATH]);
+}
+
+const computedResults = require(HEAD_COMPUTED_PATH);
+
+const sites = [];
+for (const entry of computedResults.sites) {
+  const lanternValues = entry.lantern;
+  Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key]));
+  sites.push({url: entry.url, ...lanternValues});
+}
+
+fs.writeFileSync(OUTPUT_PATH, prettyJSONStringify({sites}, {
+  tab: '  ',
+  spaceBeforeColon: '',
+  spaceInsideObject: '',
+  shouldExpand: (_, level) => level < 2,
+}));
+
+

lighthouse-core/scripts/test-lantern.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+LH_ROOT="$DIRNAME/../.."
+cd $LH_ROOT
+
+set -e
+
+# Testing lantern can be expensive, we'll only run the tests if we touched files that affect the simulations.
+CHANGED_FILES=""
+if [[ "$CI" ]]; then
+  CHANGED_FILES=$(git --no-pager diff --name-only $TRAVIS_COMMIT_RANGE)
+else
+  CHANGED_FILES=$(git --no-pager diff --name-only master)
+fi
+
+printf "Determined the following files have been touched:\n\n$CHANGED_FILES\n\n"
+
+if ! echo $CHANGED_FILES | grep -E 'dependency-graph|metrics|lantern' > /dev/null; then
+  echo "No lantern files affected, skipping lantern checks."
+  exit 0
+fi
+
+printf "Lantern files affected!\n\nDownloading test set...\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/download-traces.sh"
+
+printf "\n\nRunning lantern on all sites...\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/run-on-all-assets.js"
+
+printf "\n\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/print-correlations.js"
+
+printf "\n\nComparing to master computed values...\n"
+"$LH_ROOT/lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js"