From 3f7e5a1c966f8f80465154e4ffc59a5d8648e0c5 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Wed, 31 May 2017 11:24:10 -0700 Subject: [PATCH] Plots: make measure script more flexible (CLI args) (#2183) --- .gitignore | 2 +- plots/README.md | 40 +++++-- plots/ab-screenshot/analyze.js | 69 +++++++---- plots/bars-per-site.html | 39 ++++++ plots/bars-per-site.js | 86 ++++++++++++++ plots/index.html | 1 + plots/measure.js | 211 ++++++++++++++++----------------- plots/metrics-per-site.html | 1 + plots/sites.js | 102 ++++++++++++++++ plots/sites_subset.js | 40 +++++++ plots/utils.js | 2 +- 11 files changed, 454 insertions(+), 139 deletions(-) create mode 100644 plots/bars-per-site.html create mode 100644 plots/bars-per-site.js create mode 100644 plots/sites.js create mode 100644 plots/sites_subset.js diff --git a/.gitignore b/.gitignore index 579176547035..e7b03f4a1790 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,4 @@ lighthouse-cli/types/*.map lighthouse-core/report/partials/templates/ lighthouse-core/report/templates/*.js -plots/out/ +plots/out** diff --git a/plots/README.md b/plots/README.md index 07f45b623980..5f3399d2fdf7 100644 --- a/plots/README.md +++ b/plots/README.md @@ -14,17 +14,43 @@ You need to build lighthouse first. ### Generating & viewing charts ``` -# View all commands -$ cd plots -$ yarn run - # Run lighthouse to collect metrics data -$ yarn measure +$ node measure.js # Analyze the data to generate a summary file (i.e. out/generatedResults.js) # This will launch the charts web page in the browser -$ yarn analyze +$ node analyze.js # If you need to view the charts later -$ yarn open +$ node open.js ``` + +### Advanced usage + +``` +$ node measure.js --help + +node measure.js [options] + +Lighthouse settings: + --disable-device-emulation Disable Nexus 5X emulation [boolean] + --disable-cpu-throttling Disable CPU throttling [boolean] + --disable-network-throttling Disable network throttling [boolean] + +Options to specify sites: + --sites-path Include relative path of a json file with urls to run [default: "sites.js"] + --subset Measure a subset of popular sites + --site Include a specific site url to run + +Options: + --help Show help [boolean] + -n Number of runs per site [default: 3] + --reuse-chrome Reuse the same Chrome instance across all site runs + --keep-first-run If you use --reuse-chrome, by default the first run results are discarded + +Examples: + node measure.js -n 3 --sites-path ./sample-sites.json + node measure.js --site https://google.com/ + node measure.js --subset + +``` \ No newline at end of file diff --git a/plots/ab-screenshot/analyze.js b/plots/ab-screenshot/analyze.js index d02eb5b37f6c..af0f90cf0f0c 100644 --- a/plots/ab-screenshot/analyze.js +++ b/plots/ab-screenshot/analyze.js @@ -20,7 +20,9 @@ const fs = require('fs'); const path = require('path'); const opn = require('opn'); -const args = require('yargs').argv; +const args = require('yargs') + .default('runs', 1) + .argv; const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events'); @@ -84,25 +86,58 @@ function aggregate(outPathA, outPathB) { if (!utils.isDir(sitePathB)) { return; } - const siteScreenshotsComparison = { - siteName: siteDir, - runA: analyzeSingleRunScreenshots(sitePathA), - runB: analyzeSingleRunScreenshots(sitePathB) - }; - results.push(siteScreenshotsComparison); + + for (let i = 0; i < args.runs; i++) { + const runDirA = getRunDir(sitePathA, i); + const runDirB = getRunDir(sitePathB, i); + + const runPathA = path.resolve(sitePathA, runDirA); + const runPathB = path.resolve(sitePathB, runDirB); + + const lighthouseFileA = path.resolve(runPathA, constants.LIGHTHOUSE_RESULTS_FILENAME); + const lighthouseFileB = path.resolve(runPathB, constants.LIGHTHOUSE_RESULTS_FILENAME); + + if (!utils.isFile(lighthouseFileA) || !utils.isFile(lighthouseFileB)) { + continue; + } + + const siteScreenshotsComparison = { + siteName: `${siteDir} runA: ${runDirA} runB: ${runDirB}`, + runA: analyzeSingleRunScreenshots(runPathA), + runB: analyzeSingleRunScreenshots(runPathB), + }; + results.push(siteScreenshotsComparison); + } }); return results; } /** - * Analyzes the screenshots for the first run of a particular site. * @param {string} sitePath + * @param {number} runIndex + * @return {string} + */ +function getRunDir(sitePath, runIndex) { + return sortAndFilterRunFolders(fs.readdirSync(sitePath))[runIndex]; +} + +/** + * @param {!Array} folders + * @return {!Array} + */ +function sortAndFilterRunFolders(folders) { + return folders + .filter(folder => folder !== '.DS_Store') + .sort((a, b) => Number(a) - Number(b)); +} + +/** + * Analyzes the screenshots for the first run of a particular site. + * @param {string} runPath * @return {!SingleRunScreenshots} */ -function analyzeSingleRunScreenshots(sitePath) { - const runDir = sortAndFilterRunFolders(fs.readdirSync(sitePath))[0]; - const runPath = path.resolve(sitePath, runDir); +function analyzeSingleRunScreenshots(runPath) { const lighthouseResultsPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME); const lighthouseResults = JSON.parse(fs.readFileSync(lighthouseResultsPath)); @@ -152,18 +187,6 @@ function analyzeSingleRunScreenshots(sitePath) { } } -/** - * @param {!Array} folders - * @return {!Array} - */ -function sortAndFilterRunFolders(folders) { - return folders - .filter(folder => folder !== '.DS_Store') - .map(folder => Number(folder)) - .sort((a, b) => a - b) - .map(folder => folder.toString()); -} - /** * Marks the first screenshot that happens after a particular perf timing. * @param {SingleRunScreenshots} results diff --git a/plots/bars-per-site.html b/plots/bars-per-site.html new file mode 100644 index 000000000000..880e5ca612e8 --- /dev/null +++ b/plots/bars-per-site.html @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + diff --git a/plots/bars-per-site.js b/plots/bars-per-site.js new file mode 100644 index 000000000000..50f6fcfb9fcd --- /dev/null +++ b/plots/bars-per-site.js @@ -0,0 +1,86 @@ +/** + * @license + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +/* global Plotly, generatedResults */ +/* eslint-env browser */ + +const IGNORED_METRICS = new Set(['Navigation Start']); + +const metrics = Object.keys(generatedResults).filter(metric => !IGNORED_METRICS.has(metric)); + +let elementId = 1; + +/** + * Incrementally renders the plot, otherwise it hangs the browser + * because it's generating so many charts. + */ +const queuedPlots = []; + +function enqueuePlot(fn) { + const isFirst = queuedPlots.length == 0; + queuedPlots.push(fn); + if (isFirst) { + renderPlots(); + } +} + +function renderPlots() { + window.requestAnimationFrame(_ => { + const plotFn = queuedPlots.shift(); + if (plotFn) { + plotFn(); + renderPlots(); + } + }); +} + +function createChartElement(height = 800) { + const div = document.createElement('div'); + div.style = `width: 100%; height: ${height}px`; + div.id = 'chart' + elementId++; + document.body.appendChild(div); + return div.id; +} + +function generateGroupedBarChart() { + const sitesCount = metrics.reduce( + (acc, metric) => Math.max(acc, generatedResults[metric].length), + 0 + ); + for (let i = 0; i < sitesCount; i++) { + const data = metrics.map(metric => ({ + y: generatedResults[metric][i].metrics.map(m => m ? m.timing : null), + name: metric, + type: 'bar' + })); + + const layout = { + yaxis: { + rangemode: 'tozero' + }, + hovermode: 'closest', + barmode: 'group', + title: generatedResults[metrics[0]][i].site + }; + enqueuePlot(_ => { + Plotly.newPlot(createChartElement(), data, layout); + }); + } +} + +generateGroupedBarChart(); diff --git a/plots/index.html b/plots/index.html index ea0f99f7ff61..cd7a5ca67fa6 100644 --- a/plots/index.html +++ b/plots/index.html @@ -31,6 +31,7 @@ View results: grouped by metric grouped by site + bar charts per site diff --git a/plots/measure.js b/plots/measure.js index a6e5d7597945..1407c07421a5 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -17,11 +17,40 @@ 'use strict'; /* eslint-disable no-console */ - const path = require('path'); const parseURL = require('url').parse; const mkdirp = require('mkdirp'); +const args = require('yargs') + .wrap(Math.min(process.stdout.columns, 120)) + .help('help') + .usage('node $0 [options]') + .example('node $0 -n 3 --sites-path ./sample-sites.json') + .example('node $0 --site https://google.com/') + .example('node $0 --subset') + .describe({ + 'n': 'Number of runs per site', + 'reuse-chrome': 'Reuse the same Chrome instance across all site runs', + 'keep-first-run': 'If you use --reuse-chrome, by default the first run results are discarded', + }) + .default('n', 3) + .group( + ['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling'], + 'Lighthouse settings:') + .boolean(['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling']) + .describe({ + 'disable-device-emulation': 'Disable Nexus 5X emulation', + 'disable-cpu-throttling': 'Disable CPU throttling', + 'disable-network-throttling': 'Disable network throttling', + }) + .group(['sites-path', 'subset', 'site'], 'Options to specify sites:') + .describe({ + 'sites-path': 'Include relative path of a json file with urls to run', + 'subset': 'Measure a subset of popular sites', + 'site': 'Include a specific site url to run', + }) + .default('sites-path', 'sites.js') + .argv; const constants = require('./constants.js'); const utils = require('./utils.js'); @@ -31,135 +60,95 @@ const ChromeLauncher = require('../chrome-launcher/chrome-launcher.js'); const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); -const NUMBER_OF_RUNS = 20; - -const URLS = [ - // Flagship sites - 'https://nytimes.com', - 'https://flipkart.com', - 'http://www.espn.com/', - 'https://www.washingtonpost.com/pwa/', - - // TTI Tester sites - 'https://housing.com/in/buy/real-estate-hyderabad', - 'http://www.npr.org/', - 'http://www.vevo.com/', - 'https://weather.com/', - 'https://www.nasa.gov/', - 'https://vine.co/', - 'http://www.booking.com/', - 'http://www.thestar.com.my', - 'http://www.58pic.com', - 'http://www.dawn.com/', - 'https://www.ebs.in/IPS/', - - // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites - // (http://www.alexa.com/topsites) - // Removed adult websites and duplicates (e.g. google int'l websites) - // Also removed sites that don't have significant index pages: - // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", - // "live.com", "adf.ly", "googleusercontent.com", - - 'https://google.com', - 'https://youtube.com', - 'https://facebook.com', - 'https://baidu.com', - 'https://wikipedia.org', - 'https://yahoo.com', - 'https://amazon.com', - 'http://www.qq.com/', - 'https://taobao.com', - 'https://vk.com', - 'https://twitter.com', - 'https://instagram.com', - 'http://www.hao123.cn/', - 'http://www.sohu.com/', - 'https://sina.com.cn', - 'https://reddit.com', - 'https://linkedin.com', - 'https://tmall.com', - 'https://weibo.com', - 'https://360.cn', - 'https://yandex.ru', - 'https://ebay.com', - 'https://bing.com', - 'https://msn.com', - 'https://www.sogou.com/', - 'https://wordpress.com', - 'https://microsoft.com', - 'https://tumblr.com', - 'https://aliexpress.com', - 'https://blogspot.com', - 'https://netflix.com', - 'https://ok.ru', - 'https://stackoverflow.com', - 'https://imgur.com', - 'https://apple.com', - 'http://www.naver.com/', - 'https://mail.ru', - 'http://www.imdb.com/', - 'https://office.com', - 'https://github.com', - 'https://pinterest.com', - 'https://paypal.com', - 'http://www.tianya.cn/', - 'https://diply.com', - 'https://twitch.tv', - 'https://adobe.com', - 'https://wikia.com', - 'https://coccoc.com', - 'https://so.com', - 'https://fc2.com', - 'https://www.pixnet.net/', - 'https://dropbox.com', - 'https://zhihu.com', - 'https://whatsapp.com', - 'https://alibaba.com', - 'https://ask.com', - 'https://bbc.com' -]; +const keepFirstRun = args.keepFirstRun || !args.reuseChrome; + +function getUrls() { + if (args.site) { + return [args.site]; + } + + if (args.subset) { + return require(path.resolve(__dirname, 'sites_subset.js')); + } + + return require(path.resolve(__dirname, args.sitesPath)); +} + +const URLS = getUrls(); -/** - * Launches Chrome once at the beginning, runs all the analysis, - * and then kills Chrome. - * TODO(chenwilliam): measure the overhead of starting chrome, if it's minimal - * then open a fresh Chrome instance for each run. - */ function main() { + if (args.n === 1 && !keepFirstRun) { + console.log('ERROR: You are only doing one run and re-using chrome'); + console.log('but did not specify --keep-first-run'); + return; + } + if (utils.isDir(constants.OUT_PATH)) { console.log('ERROR: Found output from previous run at: ', constants.OUT_PATH); console.log('Please run: npm run clean'); return; } - return ChromeLauncher.launch({port: 9222}) - .then(launcher => { - return runAnalysis() + if (args.reuseChrome) { + ChromeLauncher.launch().then(launcher => { + return runAnalysisWithExistingChromeInstances(launcher) .catch(err => console.error(err)) .then(() => launcher.kill()); }); + return; + } + runAnalysisWithNewChromeInstances(); } main(); /** + * Launches a new Chrome instance for each site run. + * Returns a promise chain that analyzes all the sites n times. + * @return {!Promise} + */ +function runAnalysisWithNewChromeInstances() { + let promise = Promise.resolve(); + + for (let i = 0; i < args.n; i++) { + // Averages out any order-dependent effects such as memory pressure + utils.shuffle(URLS); + + const id = i.toString(); + const isFirstRun = i === 0; + const ignoreRun = keepFirstRun ? false : isFirstRun; + for (const url of URLS) { + promise = promise.then(() => { + return ChromeLauncher.launch().then(launcher => { + return singleRunAnalysis(url, id, launcher, {ignoreRun}) + .catch(err => console.error(err)) + .then(() => launcher.kill()); + }) + .catch(err => console.error(err)); + }); + } + } + return promise; +} + +/** + * Reuses existing Chrome instance for all site runs. * Returns a promise chain that analyzes all the sites n times. + * @param {!Launcher} launcher * @return {!Promise} */ -function runAnalysis() { +function runAnalysisWithExistingChromeInstances(launcher) { let promise = Promise.resolve(); - // Running it n + 1 times because the first run is deliberately ignored - // because it has different perf characteristics from subsequent runs - // (e.g. DNS cache which can't be easily reset between runs) - for (let i = 0; i <= NUMBER_OF_RUNS; i++) { + for (let i = 0; i < args.n; i++) { // Averages out any order-dependent effects such as memory pressure utils.shuffle(URLS); const id = i.toString(); const isFirstRun = i === 0; + const ignoreRun = keepFirstRun ? false : isFirstRun; for (const url of URLS) { - promise = promise.then(() => singleRunAnalysis(url, id, {ignoreRun: isFirstRun})); + promise = promise.then(() => singleRunAnalysis(url, id, launcher, {ignoreRun})); } } return promise; @@ -169,10 +158,11 @@ function runAnalysis() { * Analyzes a site a single time using lighthouse. * @param {string} url * @param {string} id + * @param {!Launcher} launcher * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function singleRunAnalysis(url, id, {ignoreRun}) { +function singleRunAnalysis(url, id, launcher, {ignoreRun}) { console.log('Measuring site:', url, 'run:', id); const parsedURL = parseURL(url); const urlBasedFilename = sanitizeURL(`${parsedURL.host}-${parsedURL.pathname}`); @@ -182,20 +172,27 @@ function singleRunAnalysis(url, id, {ignoreRun}) { } const outputPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME); const assetsPath = path.resolve(runPath, 'assets'); - return analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}); + return analyzeWithLighthouse(launcher, url, outputPath, assetsPath, {ignoreRun}); } /** * Runs lighthouse and save the artifacts (not used directly by plots, * but may be helpful for debugging outlier runs). + * @param {!Launcher} launcher * @param {string} url * @param {string} outputPath * @param {string} assetsPath * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) { - const flags = {output: 'json'}; +function analyzeWithLighthouse(launcher, url, outputPath, assetsPath, {ignoreRun}) { + const flags = { + output: 'json', + disableCpuThrottling: ignoreRun ? true : args.disableCpuThrottling, + disableNetworkThrottling: ignoreRun ? true : args.disableNetworkThrottling, + disableDeviceEmulation: args.disableDeviceEmulation, + port: launcher.port, + }; return lighthouse(url, flags, config) .then(lighthouseResults => { if (ignoreRun) { diff --git a/plots/metrics-per-site.html b/plots/metrics-per-site.html index 0daf43afae1c..a82146bcc073 100644 --- a/plots/metrics-per-site.html +++ b/plots/metrics-per-site.html @@ -30,6 +30,7 @@ View results: grouped by metric grouped by site + bar charts per site diff --git a/plots/sites.js b/plots/sites.js new file mode 100644 index 000000000000..2930d3a86fe7 --- /dev/null +++ b/plots/sites.js @@ -0,0 +1,102 @@ +/** + * @license + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +module.exports = [ + // Flagship sites + 'https://nytimes.com', + 'https://flipkart.com', + 'http://www.espn.com/', + 'https://www.washingtonpost.com/pwa/', + + // TTI Tester sites + 'https://housing.com/in/buy/real-estate-hyderabad', + 'http://www.npr.org/', + 'http://www.vevo.com/', + 'https://weather.com/', + 'https://www.nasa.gov/', + 'https://vine.co/', + 'http://www.booking.com/', + 'http://www.thestar.com.my', + 'http://www.58pic.com', + 'http://www.dawn.com/', + 'https://www.ebs.in/IPS/', + + // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites + // (http://www.alexa.com/topsites) + // Removed adult websites and duplicates (e.g. google int'l websites) + // Also removed sites that don't have significant index pages: + // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", + // "live.com", "adf.ly", "googleusercontent.com", + 'https://www.google.com/search?q=flowers', + 'https://youtube.com', + 'https://facebook.com', + 'https://baidu.com', + 'https://en.wikipedia.org/wiki/Google', + 'https://yahoo.com', + 'https://amazon.com', + 'http://www.qq.com/', + 'https://taobao.com', + 'https://vk.com', + 'https://mobile.twitter.com/ChromeDevTools', + 'https://www.instagram.com/stephencurry30', + 'http://www.hao123.cn/', + 'http://www.sohu.com/', + 'https://sina.com.cn', + 'https://reddit.com', + 'https://linkedin.com', + 'https://tmall.com', + 'https://weibo.com', + 'https://360.cn', + 'https://yandex.ru', + 'https://ebay.com', + 'https://bing.com', + 'https://msn.com', + 'https://www.sogou.com/', + 'https://wordpress.com', + 'https://microsoft.com', + 'https://tumblr.com', + 'https://aliexpress.com', + 'https://blogspot.com', + 'https://netflix.com', + 'https://ok.ru', + 'https://stackoverflow.com', + 'https://imgur.com', + 'https://apple.com', + 'http://www.naver.com/', + 'https://mail.ru', + 'http://www.imdb.com/', + 'https://office.com', + 'https://github.com', + 'https://pinterest.com', + 'https://paypal.com', + 'http://www.tianya.cn/', + 'https://diply.com', + 'https://twitch.tv', + 'https://adobe.com', + 'https://wikia.com', + 'https://coccoc.com', + 'https://so.com', + 'https://fc2.com', + 'https://www.pixnet.net/', + 'https://dropbox.com', + 'https://zhihu.com', + 'https://whatsapp.com', + 'https://alibaba.com', + 'https://ask.com', + 'https://bbc.com' +]; diff --git a/plots/sites_subset.js b/plots/sites_subset.js new file mode 100644 index 000000000000..a8bcbe24be2b --- /dev/null +++ b/plots/sites_subset.js @@ -0,0 +1,40 @@ +/** + * @license + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +module.exports = [ + 'https://en.wikipedia.org/wiki/Google', + 'https://mobile.twitter.com/ChromeDevTools', + 'https://www.instagram.com/stephencurry30', + 'https://amazon.com', + 'https://nytimes.com', + 'https://www.google.com/search?q=flowers', + + 'https://flipkart.com', + 'http://www.espn.com/', + 'https://www.washingtonpost.com/pwa/', + 'http://www.npr.org/', + 'http://www.booking.com/', + 'https://youtube.com', + 'https://reddit.com', + 'https://ebay.com', + 'https://stackoverflow.com', + 'https://apple.com', + + // Could not run nasa on gin3g + 'https://www.nasa.gov/', +]; diff --git a/plots/utils.js b/plots/utils.js index 5bd2b8203091..df9be4d6614e 100644 --- a/plots/utils.js +++ b/plots/utils.js @@ -55,5 +55,5 @@ function shuffle(array) { module.exports = { isDir, isFile, - shuffle + shuffle, };