ci: save perf trace to S3 on failure (#2051)

* test: save perf trace to S3 on failure * test: fix tests and reduce noisiness * feedback * add comment
GoogleChrome · Apr 25, 2017 · 4f598c5 · 4f598c5
1 parent b2eaa08
commit 4f598c5
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 9 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,16 +5,23 @@ matrix:
     - node_js: "7"
 sudo: required
 dist: trusty
+env:
+  global:
+    - ARTIFACTS_AWS_REGION=us-west-2
+    - ARTIFACTS_S3_BUCKET=lighthouse-screenshots-debugging
+    - secure: WNrQ6K8eS8/lVml6qz031oV3HQM0M3WB1+1K9DTbjlVSuYgzu3tLGCRWn67tF8+3AQWIRDXyHByONrGinWTm1teZA9SaTzuVhMrO7CHdsWTWsypQ0UqJ6H4mH/wJwJv/kW6OtQSQJGUTF8kEz33nDQVW+iFWxJx3Ow4egya6cb9D89TnUJ0FBLxYYS2DjNTbIMVBMBqSGyrd/Ww5CoYzKR96Y9jby5IId9TFFJqk3qFfXVh3Diy5TX8hUvqUpD48cfQpxE8cLZ/uOXtRsHjwMvekCSG7fNMlbQ9nJ4MmnIwhdvFrKbx1xS7g5FdRfyN+q3QcDxskytWYmtzREoRjU3N6ao8IGsXG88yXpK1bdGQ2BdNAg4IltQDGMKMzVC37sAFmXvI9S00TmWQOoAu5D4KuW9Gchc+syCpZ+umqPGePgfGryWBt0slOkFi3mixavL0kAr1kAbWbeGFBLEeX80NFFm+F2qOMuwPSsmVJ8bsgsEp0DkceyggAqb9VDyTq9ci8l/pTGTQfqbEBcXgS19BY2zDldvncfmHFisd13zHOYkqn+pi+Ta87xlskL93FOxQFn14avJo/5iNa2HmEh8vq/jBbocVr+Tt+T+OMzN/RFGBfzqU4T2K3rj9hG1cCgZFuVm+sdSuulzeUjV7y3rDAT3eAUwBnERCzwDhCpT0=
+    - secure: UdWqkZHhU2PDMKleiL386FHI6S+N0PETDd5RWPBNM/YvMREWHyWeb3vZwsMqSVri1DNk8SqI2d0PC4/jLGKchRBqlO4+smuw9f09ihn+mGI67GFsyJu6UQ5/nz5eiUKx2ZRsuB8t8BbCGiOWh+UQUuHTEgAi8ndL+ml+Vy1XRpqbT49rOTKcr+G/4J+TyEjq+wRrpZMZmhYeX0EZAcofL8KYvgbV5jbkOnQV2oI0cyCO1FQxnIz9yopvwrMlKz/1jmKiRyJOj0YaKSJ5lWs0G2RFsWmXrEAgDAlOqoJ2QCTCM5M1Fnh+0oWPholN7X4Nj/USqxRlcGaUwDLh5RhcDlrFv+wC9iBIDIay3K1/IKndikkO8OP+tCYDlx2VQRX50LPUTEQzrXEbD6pFUl1ezLQCZPejX7R/HI5rOr8WcmZzKXtl1MP0h8YX1e9HX7gXBUB0RGwnTV/P+Q6WEcxN32T/xdwbYi/YQsXfzeL7fzF5PI5LnaAc141lhcUiILs9HwnEyB7gWWLlut+GNKfse6DrNkKc3I12TVVMbHWuP2waGuc1EMcMCyC/9lb17sk5q1gbIbAsSWki9mHtrJsk0KtriKY/uOFiK5W6Dno1ITbMqbTvIelMKgQPkjYmvqyLDhADRXQz9kxlSmxcdTG5dB9lOm6ubFdRmWEPhPumq4I=
 cache:
-   directories:
-     - node_modules
-     - chrome-linux
-     - lighthouse-extension/node_modules
-     - lighthouse-viewer/node_modules
+  directories:
+    - node_modules
+    - chrome-linux
+    - lighthouse-extension/node_modules
+    - lighthouse-viewer/node_modules
 install:
   - npm install
   - npm run install-all
 before_script:
+  - gem install travis-artifacts
   - export DISPLAY=:99.0
   - export LIGHTHOUSE_CHROMIUM_PATH="$(pwd)/chrome-linux/chrome"
   - sh -e /etc/init.d/xvfb start
@@ -28,3 +35,5 @@ script:
   - npm run smokehouse
 after_success:
   - npm run coveralls
+after_failure:
+  - grep 'No screenshots' perf.json && travis-artifacts upload --path perf-0.trace.json
diff --git a/lighthouse-cli/test/smokehouse/perf/run-tests.sh b/lighthouse-cli/test/smokehouse/perf/run-tests.sh
@@ -6,8 +6,14 @@ sleep 0.5s
 
 config="lighthouse-core/config/perf.json"
 expectations="lighthouse-cli/test/smokehouse/perf/expectations.js"
+save_assets=""
 
-npm run -s smokehouse -- --config-path=$config --expectations-path=$expectations
+if [[ "$CI" = true ]]; then
+  # save assets so that failures may be examined later
+  save_assets="--save-assets-path=perf.json"
+fi
+
+npm run -s smokehouse -- --config-path=$config --expectations-path=$expectations "$save_assets"
 exit_code=$?
 
 # kill test servers

diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js
@@ -53,9 +53,10 @@ function resolveLocalOrCwd(payloadPath) {
  * Launch Chrome and do a full Lighthouse run.
  * @param {string} url
  * @param {string} configPath
+ * @param {string=} saveAssetsPath
  * @return {!LighthouseResults}
  */
-function runLighthouse(url, configPath) {
+function runLighthouse(url, configPath, saveAssetsPath) {
   const command = 'node';
   const args = [
     'lighthouse-cli/index.js',
@@ -66,6 +67,11 @@ function runLighthouse(url, configPath) {
     '--port=0'
   ];
 
+  if (saveAssetsPath) {
+    args.push('--save-assets');
+    args.push(`--output-path=${saveAssetsPath}`);
+  }
+
   // Lighthouse sometimes times out waiting to for a connection to Chrome in CI.
   // Watch for this error and retry relaunching Chrome and running Lighthouse up
   // to RETRIES times. See https://github.com/GoogleChrome/lighthouse/issues/833
@@ -89,6 +95,11 @@ function runLighthouse(url, configPath) {
     process.exit(runResults.status);
   }
 
+  if (saveAssetsPath) {
+    // If assets were saved, the JSON output was written to the specified path instead of stdout
+    return require(resolveLocalOrCwd(saveAssetsPath));
+  }
+
   return JSON.parse(runResults.stdout);
 }
 
@@ -244,7 +255,8 @@ const cli = yargs
   .help('help')
   .describe({
     'config-path': 'The path to the config JSON file',
-    'expectations-path': 'The path to the expected audit results file'
+    'expectations-path': 'The path to the expected audit results file',
+    'save-assets-path': 'Saves assets to the named path if set',
   })
   .default('config-path', DEFAULT_CONFIG_PATH)
   .default('expectations-path', DEFAULT_EXPECTATIONS_PATH)
@@ -259,7 +271,7 @@ let passingCount = 0;
 let failingCount = 0;
 expectations.forEach(expected => {
   console.log(`Checking '${expected.initialUrl}'...`);
-  const results = runLighthouse(expected.initialUrl, configPath);
+  const results = runLighthouse(expected.initialUrl, configPath, cli['save-assets-path']);
   const collated = collateResults(results, expected);
   const counts = report(collated);
   passingCount += counts.passed;