-
Notifications
You must be signed in to change notification settings - Fork 9.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
tests(lantern): add lantern regression test scripts (#5435)
- Loading branch information
1 parent
8e76467
commit b263f83
Showing
11 changed files
with
304 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
#!/usr/bin/env node | ||
/** | ||
* @license Copyright 2018 Google Inc. All Rights Reserved. | ||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
*/ | ||
'use strict'; | ||
|
||
/* eslint-disable no-console */ | ||
|
||
const fs = require('fs'); | ||
const path = require('path'); | ||
const execSync = require('child_process').execSync; | ||
const constants = require('./constants'); | ||
|
||
const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH; | ||
const HEAD_PATH = path.resolve(process.cwd(), INPUT_PATH); | ||
const MASTER_PATH = constants.MASTER_COMPUTED_PATH; | ||
|
||
const TMP_DIR = path.join(__dirname, '../../../.tmp'); | ||
const TMP_HEAD_PATH = path.join(TMP_DIR, 'HEAD-for-diff.json'); | ||
const TMP_MASTER_PATH = path.join(TMP_DIR, 'master-for-diff.json'); | ||
|
||
if (!fs.existsSync(TMP_DIR)) fs.mkdirSync(TMP_DIR); | ||
|
||
if (!fs.existsSync(HEAD_PATH) || !fs.existsSync(MASTER_PATH)) { | ||
throw new Error('Usage $0 <computed file>'); | ||
} | ||
|
||
let exitCode = 0; | ||
|
||
try { | ||
const computedResults = require(HEAD_PATH); | ||
const expectedResults = require(MASTER_PATH); | ||
|
||
const sites = []; | ||
for (const entry of computedResults.sites) { | ||
const lanternValues = entry.lantern; | ||
Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key])); | ||
sites.push({url: entry.url, ...lanternValues}); | ||
} | ||
|
||
fs.writeFileSync(TMP_HEAD_PATH, JSON.stringify({sites}, null, 2)); | ||
fs.writeFileSync(TMP_MASTER_PATH, JSON.stringify(expectedResults, null, 2)); | ||
|
||
try { | ||
execSync(`git --no-pager diff --color=always --no-index ${TMP_MASTER_PATH} ${TMP_HEAD_PATH}`); | ||
console.log('✅ PASS No changes between expected and computed!'); | ||
} catch (err) { | ||
console.log('❌ FAIL Changes between expected and computed!\n'); | ||
console.log(err.stdout.toString()); | ||
exitCode = 1; | ||
} | ||
} finally { | ||
fs.unlinkSync(TMP_HEAD_PATH); | ||
fs.unlinkSync(TMP_MASTER_PATH); | ||
} | ||
|
||
process.exit(exitCode); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
/** | ||
* @license Copyright 2018 Google Inc. All Rights Reserved. | ||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
*/ | ||
'use strict'; | ||
|
||
const path = require('path'); | ||
|
||
/* eslint-disable max-len */ | ||
|
||
module.exports = { | ||
SITE_INDEX_WITH_GOLDEN_PATH: './lantern-data/site-index-plus-golden-expectations.json', | ||
SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH: path.join(__dirname, '../../../.tmp/site-index-plus-golden-expectations-plus-computed.json'), | ||
MASTER_COMPUTED_PATH: path.join(__dirname, '../../test/fixtures/lantern-master-computed-values.json'), | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,13 +1,22 @@ | ||
#!/bin/bash | ||
|
||
set -e | ||
|
||
DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" | ||
LH_ROOT_PATH="$DIRNAME/../../.." | ||
cd $LH_ROOT_PATH | ||
|
||
if [[ -f lantern-data/site-index-plus-golden-expectations.json ]] && ! [[ "$FORCE" ]]; then | ||
echo "Lantern data already detected, done." | ||
exit 0 | ||
fi | ||
|
||
rm -rf lantern-data/ | ||
mkdir lantern-data/ && cd lantern-data/ | ||
|
||
# snapshot of ~100 traces with no throttling recorded 2017-12-06 on a HP z840 workstation | ||
TAR_URL="https://drive.google.com/a/chromium.org/uc?id=1_w2g6fQVLgHI62FApsyUDejZyHNXMLm0&export=download" | ||
curl -o lantern-traces.tar.gz -L $TAR_URL | ||
|
||
tar -xzf lantern-traces.tar.gz | ||
mv lantern-traces-subset lantern-data | ||
rm lantern-traces.tar.gz |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 44 additions & 0 deletions
44
lighthouse-core/scripts/lantern/update-master-lantern-values.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
#!/usr/bin/env node | ||
/** | ||
* @license Copyright 2018 Google Inc. All Rights Reserved. | ||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
*/ | ||
'use strict'; | ||
|
||
/* eslint-disable no-console */ | ||
|
||
const fs = require('fs'); | ||
const path = require('path'); | ||
const execFileSync = require('child_process').execFileSync; | ||
const prettyJSONStringify = require('pretty-json-stringify'); | ||
const constants = require('./constants'); | ||
|
||
const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_PATH; | ||
const SITE_INDEX_PATH = path.resolve(process.cwd(), INPUT_PATH); | ||
const HEAD_COMPUTED_PATH = constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH; | ||
const RUN_ALL_SCRIPT_PATH = path.join(__dirname, 'run-on-all-assets.js'); | ||
const OUTPUT_PATH = constants.MASTER_COMPUTED_PATH; | ||
|
||
if (!fs.existsSync(HEAD_COMPUTED_PATH) || process.env.FORCE) { | ||
if (!fs.existsSync(SITE_INDEX_PATH)) throw new Error('Usage $0 <expectations file>'); | ||
execFileSync(RUN_ALL_SCRIPT_PATH, [SITE_INDEX_PATH]); | ||
} | ||
|
||
const computedResults = require(HEAD_COMPUTED_PATH); | ||
|
||
const sites = []; | ||
for (const entry of computedResults.sites) { | ||
const lanternValues = entry.lantern; | ||
Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key])); | ||
sites.push({url: entry.url, ...lanternValues}); | ||
} | ||
|
||
fs.writeFileSync(OUTPUT_PATH, prettyJSONStringify({sites}, { | ||
tab: ' ', | ||
spaceBeforeColon: '', | ||
spaceInsideObject: '', | ||
shouldExpand: (_, level) => level < 2, | ||
})); | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
#!/usr/bin/env bash | ||
|
||
DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" | ||
LH_ROOT="$DIRNAME/../.." | ||
cd $LH_ROOT | ||
|
||
set -e | ||
|
||
# Testing lantern can be expensive, we'll only run the tests if we touched files that affect the simulations. | ||
CHANGED_FILES="" | ||
if [[ "$CI" ]]; then | ||
CHANGED_FILES=$(git --no-pager diff --name-only $TRAVIS_COMMIT_RANGE) | ||
else | ||
CHANGED_FILES=$(git --no-pager diff --name-only master) | ||
fi | ||
|
||
printf "Determined the following files have been touched:\n\n$CHANGED_FILES\n\n" | ||
|
||
if ! echo $CHANGED_FILES | grep -E 'dependency-graph|metrics|lantern' > /dev/null; then | ||
echo "No lantern files affected, skipping lantern checks." | ||
exit 0 | ||
fi | ||
|
||
printf "Lantern files affected!\n\nDownloading test set...\n" | ||
"$LH_ROOT/lighthouse-core/scripts/lantern/download-traces.sh" | ||
|
||
printf "\n\nRunning lantern on all sites...\n" | ||
"$LH_ROOT/lighthouse-core/scripts/lantern/run-on-all-assets.js" | ||
|
||
printf "\n\n" | ||
"$LH_ROOT/lighthouse-core/scripts/lantern/print-correlations.js" | ||
|
||
printf "\n\nComparing to master computed values...\n" | ||
"$LH_ROOT/lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js" |
Oops, something went wrong.