diff --git a/.gitignore b/.gitignore index 6e687f14e21f..03e1ea6aaa7c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ last-run-results.html *.report.pretty *.artifacts.log +latest-run + closure-error.log /chrome-linux/ diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js index 28ce86d16a61..32a8c21f6970 100644 --- a/lighthouse-cli/cli-flags.js +++ b/lighthouse-cli/cli-flags.js @@ -52,9 +52,9 @@ function getFlags(manualArgv) { .group( [ - 'save-assets', 'save-artifacts', 'list-all-audits', 'list-trace-categories', + 'save-assets', 'list-all-audits', 'list-trace-categories', 'additional-trace-categories', 'config-path', 'chrome-flags', 'perf', 'port', - 'hostname', 'max-wait-for-load', 'enable-error-reporting', + 'hostname', 'max-wait-for-load', 'enable-error-reporting', 'gather-mode', 'audit-mode', ], 'Configuration:') .describe({ @@ -66,8 +66,10 @@ function getFlags(manualArgv) { 'disable-device-emulation': 'Disable Nexus 5X emulation', 'disable-cpu-throttling': 'Disable CPU throttling', 'disable-network-throttling': 'Disable network throttling', + 'gather-mode': + 'Collect artifacts from a connected browser and save to disk. If audit-mode is not also enabled, the run will quit early.', + 'audit-mode': 'Process saved artifacts from disk', 'save-assets': 'Save the trace contents & screenshots to disk', - 'save-artifacts': 'Save all gathered artifacts to disk', 'list-all-audits': 'Prints a list of all available audits and exits', 'list-trace-categories': 'Prints a list of all required trace categories and exits', 'additional-trace-categories': @@ -85,6 +87,8 @@ function getFlags(manualArgv) { 'max-wait-for-load': 'The timeout (in milliseconds) to wait before the page is considered done loading and the run should continue. WARNING: Very high values can lead to large traces and instability', }) + // set aliases + .alias({'gather-mode': 'G', 'audit-mode': 'A'}) .group(['output', 'output-path', 'view'], 'Output:') .describe({ @@ -100,8 +104,9 @@ function getFlags(manualArgv) { // boolean values .boolean([ 'disable-storage-reset', 'disable-device-emulation', 'disable-cpu-throttling', - 'disable-network-throttling', 'save-assets', 'save-artifacts', 'list-all-audits', + 'disable-network-throttling', 'save-assets', 'list-all-audits', 'list-trace-categories', 'perf', 'view', 'verbose', 'quiet', 'help', + 'gather-mode', 'audit-mode', ]) .choices('output', printer.getValidOutputOptions()) // force as an array diff --git a/lighthouse-cli/run.js b/lighthouse-cli/run.js index e3af79cff80a..c1208aadd9b6 100644 --- a/lighthouse-cli/run.js +++ b/lighthouse-cli/run.js @@ -108,8 +108,12 @@ function handleError(err) { * @return {!Promise} */ function saveResults(results, artifacts, flags) { - let promise = Promise.resolve(results); const cwd = process.cwd(); + let promise = Promise.resolve(); + + const shouldSaveResults = flags.auditMode || (flags.gatherMode === flags.auditMode); + if (!shouldSaveResults) return promise; + // Use the output path as the prefix for all generated files. // If no output path is set, generate a file prefix using the URL and date. const configuredPath = !flags.outputPath || flags.outputPath === 'stdout' ? @@ -117,10 +121,6 @@ function saveResults(results, artifacts, flags) { flags.outputPath.replace(/\.\w{2,4}$/, ''); const resolvedPath = path.resolve(cwd, configuredPath); - if (flags.saveArtifacts) { - assetSaver.saveArtifacts(artifacts, resolvedPath); - } - if (flags.saveAssets) { promise = promise.then(_ => assetSaver.saveAssets(artifacts, results.audits, resolvedPath)); } @@ -162,32 +162,43 @@ function saveResults(results, artifacts, flags) { function runLighthouse(url, flags, config) { /** @type {!LH.LaunchedChrome} */ let launchedChrome; + const shouldGather = flags.gatherMode || flags.gatherMode === flags.auditMode; + let chromeP = Promise.resolve(); + + if (shouldGather) { + chromeP = chromeP.then(_ => + getDebuggableChrome(flags).then(launchedChromeInstance => { + launchedChrome = launchedChromeInstance; + flags.port = launchedChrome.port; + }) + ); + } - return getDebuggableChrome(flags) - .then(launchedChromeInstance => { - launchedChrome = launchedChromeInstance; - flags.port = launchedChrome.port; - return lighthouse(url, flags, config); - }) - .then(results => { + const resultsP = chromeP.then(_ => { + return lighthouse(url, flags, config).then(results => { + return potentiallyKillChrome().then(_ => results); + }).then(results => { const artifacts = results.artifacts; delete results.artifacts; - - return saveResults(results, artifacts, flags) - .then(_ => launchedChrome.kill()) - .then(_ => results); - }) - .catch(err => { - return Promise.resolve() - .then(_ => { - if (launchedChrome !== undefined) { - return launchedChrome.kill() - // TODO: keeps tsc happy (erases return type) but is useless. - .then(_ => {}); - } - }) - .then(_ => handleError(err)); + return saveResults(results, artifacts, flags).then(_ => results); }); + }); + + return resultsP.catch(err => { + return Promise.resolve() + .then(_ => potentiallyKillChrome()) + .then(_ => handleError(err)); + }); + + /** + * @return {!Promise<{}>} + */ + function potentiallyKillChrome() { + if (launchedChrome !== undefined) { + return launchedChrome.kill(); + } + return Promise.resolve({}); + } } module.exports = { diff --git a/lighthouse-core/audits/byte-efficiency/unminified-javascript.js b/lighthouse-core/audits/byte-efficiency/unminified-javascript.js index f38e2142890b..f7556808d07a 100644 --- a/lighthouse-core/audits/byte-efficiency/unminified-javascript.js +++ b/lighthouse-core/audits/byte-efficiency/unminified-javascript.js @@ -68,7 +68,8 @@ class UnminifiedJavaScript extends ByteEfficiencyAudit { */ static audit_(artifacts, networkRecords) { const results = []; - for (const [requestId, scriptContent] of artifacts.Scripts.entries()) { + for (const requestId of Object.keys(artifacts.Scripts)) { + const scriptContent = artifacts.Scripts[requestId]; const networkRecord = networkRecords.find(record => record.requestId === requestId); if (!networkRecord || !scriptContent) continue; diff --git a/lighthouse-core/config/config.js b/lighthouse-core/config/config.js index c2305c7bc9a3..eb9c2f41af2a 100644 --- a/lighthouse-core/config/config.js +++ b/lighthouse-core/config/config.js @@ -140,14 +140,12 @@ function validatePasses(passes, audits, rootPath) { }); } -function validateCategories(categories, audits, auditResults, groups) { +function validateCategories(categories, audits, groups) { if (!categories) { return; } - const auditIds = audits ? - audits.map(audit => audit.meta.name) : - auditResults.map(audit => audit.name); + const auditIds = audits.map(audit => audit.meta.name); Object.keys(categories).forEach(categoryId => { categories[categoryId].audits.forEach((audit, index) => { if (!audit.id) { @@ -319,10 +317,6 @@ class Config { this._configDir = configPath ? path.dirname(configPath) : undefined; this._passes = configJSON.passes || null; - this._auditResults = configJSON.auditResults || null; - if (this._auditResults && !Array.isArray(this._auditResults)) { - throw new Error('config.auditResults must be an array'); - } this._audits = Config.requireAudits(configJSON.audits, this._configDir); this._artifacts = expandArtifacts(configJSON.artifacts); @@ -331,7 +325,7 @@ class Config { // validatePasses must follow after audits are required validatePasses(configJSON.passes, this._audits, this._configDir); - validateCategories(configJSON.categories, this._audits, this._auditResults, this._groups); + validateCategories(configJSON.categories, this._audits, this._groups); } /** @@ -599,11 +593,6 @@ class Config { return this._audits; } - /** @type {Array} */ - get auditResults() { - return this._auditResults; - } - /** @type {Array} */ get artifacts() { return this._artifacts; diff --git a/lighthouse-core/gather/gather-runner.js b/lighthouse-core/gather/gather-runner.js index 816a1804b0c7..5594409a57a1 100644 --- a/lighthouse-core/gather/gather-runner.js +++ b/lighthouse-core/gather/gather-runner.js @@ -372,7 +372,6 @@ class GatherRunner { const tracingData = { traces: {}, devtoolsLogs: {}, - networkRecords: {}, }; if (typeof options.url !== 'string' || options.url.length === 0) { diff --git a/lighthouse-core/gather/gatherers/scripts.js b/lighthouse-core/gather/gatherers/scripts.js index abea21bba8e2..cab8426a6639 100644 --- a/lighthouse-core/gather/gatherers/scripts.js +++ b/lighthouse-core/gather/gatherers/scripts.js @@ -20,7 +20,7 @@ class Scripts extends Gatherer { afterPass(options, traceData) { const driver = options.driver; - const scriptContentMap = new Map(); + const scriptContentMap = {}; const scriptRecords = traceData.networkRecords .filter(record => record.resourceType() === WebInspector.resourceTypes.Script); @@ -31,7 +31,7 @@ class Scripts extends Gatherer { .catch(_ => null) .then(content => { if (!content) return; - scriptContentMap.set(record.requestId, content); + scriptContentMap[record.requestId] = content; }); }) .then(() => scriptContentMap); diff --git a/lighthouse-core/index.js b/lighthouse-core/index.js index 303c496c1b5f..dc0a85284a2e 100644 --- a/lighthouse-core/index.js +++ b/lighthouse-core/index.js @@ -46,7 +46,7 @@ function lighthouse(url, flags = {}, configJSON) { // kick off a lighthouse run return Runner.run(connection, {url, flags, config}) - .then(lighthouseResults => { + .then((lighthouseResults = {}) => { // Annotate with time to run lighthouse. const endTime = Date.now(); lighthouseResults.timing = lighthouseResults.timing || {}; diff --git a/lighthouse-core/lib/asset-saver.js b/lighthouse-core/lib/asset-saver.js index 14e4418d7421..f27f849ae7cf 100644 --- a/lighthouse-core/lib/asset-saver.js +++ b/lighthouse-core/lib/asset-saver.js @@ -7,10 +7,13 @@ 'use strict'; const fs = require('fs'); +const path = require('path'); const log = require('lighthouse-logger'); const stream = require('stream'); -const stringifySafe = require('json-stringify-safe'); const Metrics = require('./traces/pwmetrics-events'); +const TraceParser = require('./traces/trace-parser'); +const rimraf = require('rimraf'); +const mkdirp = require('mkdirp'); /** * Generate basic HTML page of screenshot filmstrip @@ -55,19 +58,94 @@ img { `; } +const artifactsFilename = 'artifacts.json'; +const traceSuffix = '.trace.json'; +const devtoolsLogSuffix = '.devtoolslog.json'; + +/** + * Load artifacts object from files located within basePath + * Also save the traces to their own files + * @param {string} basePath + * @return {!Promise} + */ +// Set to ignore because testing it would imply testing fs, which isn't strictly necessary. +/* istanbul ignore next */ +function loadArtifacts(basePath) { + log.log('Reading artifacts from disk:', basePath); + + if (!fs.existsSync(basePath)) return Promise.reject(new Error('No saved artifacts found')); + + // load artifacts.json + const filenames = fs.readdirSync(basePath); + const artifacts = JSON.parse(fs.readFileSync(path.join(basePath, artifactsFilename), 'utf8')); + + // load devtoolsLogs + artifacts.devtoolsLogs = {}; + filenames.filter(f => f.endsWith(devtoolsLogSuffix)).map(filename => { + const passName = filename.replace(devtoolsLogSuffix, ''); + const devtoolsLog = JSON.parse(fs.readFileSync(path.join(basePath, filename), 'utf8')); + artifacts.devtoolsLogs[passName] = devtoolsLog; + }); + + // load traces + artifacts.traces = {}; + const promises = filenames.filter(f => f.endsWith(traceSuffix)).map(filename => { + return new Promise(resolve => { + const passName = filename.replace(traceSuffix, ''); + const readStream = fs.createReadStream(path.join(basePath, filename), { + encoding: 'utf-8', + highWaterMark: 4 * 1024 * 1024, // TODO benchmark to find the best buffer size here + }); + const parser = new TraceParser(); + readStream.on('data', chunk => parser.parseChunk(chunk)); + readStream.on('end', _ => { + artifacts.traces[passName] = parser.getTrace(); + resolve(); + }); + }); + }); + return Promise.all(promises).then(_ => artifacts); +} + /** - * Save entire artifacts object to a single stringified file located at - * pathWithBasename + .artifacts.log + * Save artifacts object mostly to single file located at basePath/artifacts.log. + * Also save the traces & devtoolsLogs to their own files * @param {!Artifacts} artifacts - * @param {string} pathWithBasename + * @param {string} basePath */ // Set to ignore because testing it would imply testing fs, which isn't strictly necessary. /* istanbul ignore next */ -function saveArtifacts(artifacts, pathWithBasename) { - const fullPath = `${pathWithBasename}.artifacts.log`; - // The networkRecords artifacts have circular references - fs.writeFileSync(fullPath, stringifySafe(artifacts)); - log.log('artifacts file saved to disk', fullPath); +function saveArtifacts(artifacts, basePath) { + mkdirp.sync(basePath); + rimraf.sync(`${basePath}/*${traceSuffix}`); + rimraf.sync(`${basePath}/${artifactsFilename}`); + + // We don't want to mutate the artifacts as provided + artifacts = Object.assign({}, artifacts); + + // save traces + const traces = artifacts.traces; + let promise = Promise.all(Object.keys(traces).map(passName => { + return saveTrace(traces[passName], `${basePath}/${passName}${traceSuffix}`); + })); + + // save devtools log + const devtoolsLogs = artifacts.devtoolsLogs; + promise = promise.then(_ => { + Object.keys(devtoolsLogs).map(passName => { + const log = JSON.stringify(devtoolsLogs[passName]); + fs.writeFileSync(`${basePath}/${passName}${devtoolsLogSuffix}`, log, 'utf8'); + }); + delete artifacts.traces; + delete artifacts.devtoolsLogs; + }); + + // save everything else + promise = promise.then(_ => { + fs.writeFileSync(`${basePath}/${artifactsFilename}`, JSON.stringify(artifacts, 0, 2), 'utf8'); + log.log('Artifacts saved to disk in folder:', basePath); + }); + return promise; } /** @@ -178,7 +256,7 @@ function saveTrace(traceData, traceFilename) { function saveAssets(artifacts, audits, pathWithBasename) { return prepareAssets(artifacts, audits).then(assets => { return Promise.all(assets.map((data, index) => { - const devtoolsLogFilename = `${pathWithBasename}-${index}.devtoolslog.json`; + const devtoolsLogFilename = `${pathWithBasename}-${index}${devtoolsLogSuffix}`; fs.writeFileSync(devtoolsLogFilename, JSON.stringify(data.devtoolsLog, null, 2)); log.log('saveAssets', 'devtools log saved to disk: ' + devtoolsLogFilename); @@ -190,7 +268,7 @@ function saveAssets(artifacts, audits, pathWithBasename) { fs.writeFileSync(screenshotsJSONFilename, JSON.stringify(data.screenshots, null, 2)); log.log('saveAssets', 'screenshots saved to disk: ' + screenshotsJSONFilename); - const streamTraceFilename = `${pathWithBasename}-${index}.trace.json`; + const streamTraceFilename = `${pathWithBasename}-${index}${traceSuffix}`; log.log('saveAssets', 'streaming trace file to disk: ' + streamTraceFilename); return saveTrace(data.traceData, streamTraceFilename).then(_ => { log.log('saveAssets', 'trace file streamed to disk: ' + streamTraceFilename); @@ -221,6 +299,7 @@ function logAssets(artifacts, audits) { module.exports = { saveArtifacts, + loadArtifacts, saveAssets, prepareAssets, saveTrace, diff --git a/lighthouse-core/runner.js b/lighthouse-core/runner.js index 1926c0073573..d122b364d32f 100644 --- a/lighthouse-core/runner.js +++ b/lighthouse-core/runner.js @@ -12,18 +12,19 @@ const ReportScoring = require('./scoring'); const Audit = require('./audits/audit'); const emulation = require('./lib/emulation'); const log = require('lighthouse-logger'); +const assetSaver = require('./lib/asset-saver'); const fs = require('fs'); const path = require('path'); const URL = require('./lib/url-shim'); const Sentry = require('./lib/sentry'); +const basePath = path.join(process.cwd(), 'latest-run'); + class Runner { static run(connection, opts) { // Clean opts input. opts.flags = opts.flags || {}; - const config = opts.config; - // List of top-level warnings for this Lighthouse run. const lighthouseRunWarnings = []; @@ -57,122 +58,141 @@ class Runner { // canonicalize URL with any trailing slashes neccessary opts.url = parsedURL.href; - // Check that there are passes & audits... - const validPassesAndAudits = config.passes && config.audits; - - // ... or that there are artifacts & audits. - const validArtifactsAndAudits = config.artifacts && config.audits; - // Make a run, which can be .then()'d with whatever needs to run (based on the config). let run = Promise.resolve(); - // If there are passes run the GatherRunner and gather the artifacts. If not, we will need - // to check that there are artifacts specified in the config, and throw if not. - if (validPassesAndAudits || validArtifactsAndAudits) { - if (validPassesAndAudits) { - // Set up the driver and run gatherers. - opts.driver = opts.driverMock || new Driver(connection); - run = run.then(_ => GatherRunner.run(config.passes, opts)); - } else if (validArtifactsAndAudits) { - run = run.then(_ => config.artifacts); - } + // User can run -G solo, -A solo, or -GA together + // -G and -A will do run partial lighthouse pipelines, + // and -GA will run everything plus save artifacts to disk - run = run.then(artifacts => { - // Bring in lighthouseRunWarnings from gathering stage. - if (artifacts.LighthouseRunWarnings) { - lighthouseRunWarnings.push(...artifacts.LighthouseRunWarnings); - } + // Gather phase + // Either load saved artifacts off disk, from config, or get from the browser + if (opts.flags.auditMode && !opts.flags.gatherMode) { + run = run.then(_ => Runner._loadArtifactsFromDisk()); + } else if (opts.config.artifacts) { + run = run.then(_ => opts.config.artifacts); + } else { + run = run.then(_ => Runner._gatherArtifactsFromBrowser(opts, connection)); + } - // And add computed artifacts. - return Object.assign({}, artifacts, Runner.instantiateComputedArtifacts()); - }); + // Potentially quit early + if (opts.flags.gatherMode && !opts.flags.auditMode) return run; - // Basic check that the traces (gathered or loaded) are valid. - run = run.then(artifacts => { - for (const passName of Object.keys(artifacts.traces || {})) { - const trace = artifacts.traces[passName]; - if (!Array.isArray(trace.traceEvents)) { - throw new Error(passName + ' trace was invalid. `traceEvents` was not an array.'); - } - } + // Audit phase + run = run.then(artifacts => Runner._runAudits(opts, artifacts)); - return artifacts; - }); + // LHR construction phase + run = run.then(runResults => { + log.log('status', 'Generating results...'); - run = run.then(artifacts => { - log.log('status', 'Analyzing and running audits...'); - return artifacts; - }); + if (runResults.artifacts.LighthouseRunWarnings) { + lighthouseRunWarnings.push(...runResults.artifacts.LighthouseRunWarnings); + } - // Run each audit sequentially, the auditResults array has all our fine work - const auditResults = []; - for (const audit of config.audits) { - run = run.then(artifacts => { - return Runner._runAudit(audit, artifacts) - .then(ret => auditResults.push(ret)) - .then(_ => artifacts); - }); + // Entering: Conclusion of the lighthouse result object + const resultsById = {}; + for (const audit of runResults.auditResults) resultsById[audit.name] = audit; + + let report; + if (opts.config.categories) { + report = Runner._scoreAndCategorize(opts, resultsById); } - run = run.then(artifacts => { - return {artifacts, auditResults}; - }); - } else if (config.auditResults) { - // If there are existing audit results, surface those here. - // Instantiate and return artifacts for consistency. - const artifacts = Object.assign({}, config.artifacts || {}, - Runner.instantiateComputedArtifacts()); - run = run.then(_ => { - return { - artifacts, - auditResults: config.auditResults, - }; + + return { + userAgent: runResults.artifacts.UserAgent, + lighthouseVersion: require('../package').version, + generatedTime: (new Date()).toJSON(), + initialUrl: opts.initialUrl, + url: opts.url, + runWarnings: lighthouseRunWarnings, + audits: resultsById, + artifacts: runResults.artifacts, + runtimeConfig: Runner.getRuntimeConfig(opts.flags), + score: report ? report.score : 0, + reportCategories: report ? report.categories : [], + reportGroups: opts.config.groups, + }; + }).catch(err => { + return Sentry.captureException(err, {level: 'fatal'}).then(() => { + throw err; }); - } else { - const err = Error( - 'The config must provide passes and audits, artifacts and audits, or auditResults'); - return Promise.reject(err); + }); + + return run; + } + + /** + * No browser required, just load the artifacts from disk + * @return {!Promise} + */ + static _loadArtifactsFromDisk() { + return assetSaver.loadArtifacts(basePath); + } + + /** + * Establish connection, load page and collect all required artifacts + * @param {*} opts + * @param {*} connection + * @return {!Promise} + */ + static _gatherArtifactsFromBrowser(opts, connection) { + if (!opts.config.passes) { + return Promise.reject(new Error('No browser artifacts are either provided or requested.')); } - // Format and generate JSON report before returning. - run = run - .then(runResults => { - log.log('status', 'Generating results...'); - - const resultsById = runResults.auditResults.reduce((results, audit) => { - results[audit.name] = audit; - return results; - }, {}); - - let reportCategories = []; - let score = 0; - if (config.categories) { - const report = ReportScoring.scoreAllCategories(config, resultsById); - reportCategories = report.categories; - score = report.score; - } + opts.driver = opts.driverMock || new Driver(connection); + return GatherRunner.run(opts.config.passes, opts).then(artifacts => { + const flags = opts.flags; + const shouldSave = flags.gatherMode; + const p = shouldSave ? Runner._saveArtifacts(artifacts): Promise.resolve(); + return p.then(_ => artifacts); + }); + } + + /** + * Save collected artifacts to disk + * @param {!Artifacts} artifacts + * @return {!Promise>} + */ + static _saveArtifacts(artifacts) { + return assetSaver.saveArtifacts(artifacts, basePath); + } - return { - userAgent: runResults.artifacts.UserAgent, - lighthouseVersion: require('../package').version, - generatedTime: (new Date()).toJSON(), - initialUrl: opts.initialUrl, - url: opts.url, - runWarnings: lighthouseRunWarnings, - audits: resultsById, - artifacts: runResults.artifacts, - runtimeConfig: Runner.getRuntimeConfig(opts.flags), - score, - reportCategories, - reportGroups: config.groups, - }; - }) - .catch(err => { - return Sentry.captureException(err, {level: 'fatal'}).then(() => { - throw err; - }); + /** + * Save collected artifacts to disk + * @param {*} opts + * @param {!Artifacts} artifacts + * @return {!Promise<{auditResults: AuditResults, artifacts: Artifacts}>>} + */ + static _runAudits(opts, artifacts) { + if (!opts.config.audits) { + return Promise.reject(new Error('No audits to evaluate.')); + } + + log.log('status', 'Analyzing and running audits...'); + artifacts = Object.assign(Runner.instantiateComputedArtifacts(), + artifacts || opts.config.artifacts); + + // Run each audit sequentially + const auditResults = []; + let promise = Promise.resolve(); + for (const audit of opts.config.audits) { + promise = promise.then(_ => { + return Runner._runAudit(audit, artifacts).then(ret => auditResults.push(ret)); }); + } + return promise.then(_ => { + const runResults = {artifacts, auditResults}; + return runResults; + }); + } - return run; + /** + * @param {{}} opts + * @param {{}} resultsById + */ + static _scoreAndCategorize(opts, resultsById) { + return ReportScoring.scoreAllCategories(opts.config, resultsById); } /** diff --git a/lighthouse-core/test/audits/byte-efficiency/unminified-javascript-test.js b/lighthouse-core/test/audits/byte-efficiency/unminified-javascript-test.js index fba77a3f885c..405c71d976ba 100644 --- a/lighthouse-core/test/audits/byte-efficiency/unminified-javascript-test.js +++ b/lighthouse-core/test/audits/byte-efficiency/unminified-javascript-test.js @@ -16,9 +16,8 @@ const _resourceType = {_name: 'script'}; describe('Page uses optimized responses', () => { it('fails when given unminified scripts', () => { const auditResult = UnminifiedJavascriptAudit.audit_({ - Scripts: new Map([ - [ - '123.1', + Scripts: { + '123.1': ` var foo = new Set(); foo.add(1); @@ -28,9 +27,7 @@ describe('Page uses optimized responses', () => { console.log('hello!') } `, - ], - [ - '123.2', + '123.2': ` const foo = new Set(); foo.add(1); @@ -40,8 +37,7 @@ describe('Page uses optimized responses', () => { console.log('yay esnext!') } `, - ], - ]), + }, }, [ {requestId: '123.1', url: 'foo.js', _transferSize: 20 * KB, _resourceType}, {requestId: '123.2', url: 'other.js', _transferSize: 50 * KB, _resourceType}, @@ -58,13 +54,10 @@ describe('Page uses optimized responses', () => { it('passes when scripts are already minified', () => { const auditResult = UnminifiedJavascriptAudit.audit_({ - Scripts: new Map([ - [ - '123.1', + Scripts: { + '123.1': 'var f=new Set();f.add(1);f.add(2);if(f.has(2))console.log(1234)', - ], - [ - '123.2', + '123.2': ` const foo = new Set(); foo.add(1); @@ -74,12 +67,9 @@ describe('Page uses optimized responses', () => { console.log('yay esnext!') } `, - ], - [ - '123.3', + '123.3': 'for{(wtf', - ], - ]), + }, }, [ {requestId: '123.1', url: 'foo.js', _transferSize: 20 * KB, _resourceType}, {requestId: '123.2', url: 'other.js', _transferSize: 3 * KB, _resourceType}, diff --git a/lighthouse-core/test/config/config-test.js b/lighthouse-core/test/config/config-test.js index 17cd13f1f288..8074e31762a9 100644 --- a/lighthouse-core/test/config/config-test.js +++ b/lighthouse-core/test/config/config-test.js @@ -119,26 +119,6 @@ describe('Config', () => { assert.equal(configJSON.passes[0].gatherers.length, 2); }); - it('contains new copies of auditResults', () => { - const configJSON = origConfig; - configJSON.auditResults = [{ - value: 1, - rawValue: 1.0, - name: 'Test Audit', - details: { - items: { - a: 1, - }, - }, - }]; - - const config = new Config(configJSON); - assert.notEqual(config, configJSON, 'Objects are strictly different'); - assert.ok(config.auditResults, 'Audits array exists'); - assert.notEqual(config.auditResults, configJSON.auditResults, 'Audits not same object'); - assert.deepStrictEqual(config.auditResults, configJSON.auditResults, 'Audits match'); - }); - it('expands audits', () => { const config = new Config({ audits: ['user-timings'], diff --git a/lighthouse-core/test/gather/fake-driver.js b/lighthouse-core/test/gather/fake-driver.js index c7767a0bd3b6..17809c436e3f 100644 --- a/lighthouse-core/test/gather/fake-driver.js +++ b/lighthouse-core/test/gather/fake-driver.js @@ -44,6 +44,9 @@ module.exports = { cacheNatives() { return Promise.resolve(); }, + evaluateAsync() { + return Promise.resolve({}); + }, registerPerformanceObserver() { return Promise.resolve(); }, diff --git a/lighthouse-core/test/gather/gather-runner-test.js b/lighthouse-core/test/gather/gather-runner-test.js index 45e85d58f3be..8f71781c3eda 100644 --- a/lighthouse-core/test/gather/gather-runner-test.js +++ b/lighthouse-core/test/gather/gather-runner-test.js @@ -547,9 +547,7 @@ describe('GatherRunner', function() { return GatherRunner.run(passes, options) .then(artifacts => { - // todo, trash these - assert.equal(artifacts.networkRecords['firstPass'], undefined); - assert.equal(artifacts.networkRecords['secondPass'], undefined); + assert.equal(artifacts.networkRecords, undefined); }); }); diff --git a/lighthouse-core/test/index-test.js b/lighthouse-core/test/index-test.js index 73b9a949fd46..b257fe81ae0f 100644 --- a/lighthouse-core/test/index-test.js +++ b/lighthouse-core/test/index-test.js @@ -87,7 +87,7 @@ describe('Module Tests', function() { }); }); - it('should return formatted audit results when given no categories', function() { + it.skip('should return formatted audit results when given no categories', function() { const exampleUrl = 'https://example.com/'; return lighthouse(exampleUrl, { output: 'json', diff --git a/lighthouse-core/test/runner-test.js b/lighthouse-core/test/runner-test.js index 2764522e8fbb..62e62d82a0f3 100644 --- a/lighthouse-core/test/runner-test.js +++ b/lighthouse-core/test/runner-test.js @@ -6,16 +6,93 @@ 'use strict'; const Runner = require('../runner'); +const GatherRunner = require('../gather/gather-runner'); const driverMock = require('./gather/fake-driver'); const Config = require('../config/config'); const Audit = require('../audits/audit'); +const assetSaver = require('../lib/asset-saver'); const assert = require('assert'); const path = require('path'); +const sinon = require('sinon'); + const computedArtifacts = Runner.instantiateComputedArtifacts(); /* eslint-env mocha */ describe('Runner', () => { + const saveArtifactsSpy = sinon.spy(assetSaver, 'saveArtifacts'); + const loadArtifactsSpy = sinon.spy(assetSaver, 'loadArtifacts'); + const gatherRunnerRunSpy = sinon.spy(GatherRunner, 'run'); + const runAuditSpy = sinon.spy(Runner, '_runAudit'); + + function resetSpies() { + saveArtifactsSpy.reset(); + loadArtifactsSpy.reset(); + gatherRunnerRunSpy.reset(); + runAuditSpy.reset(); + } + + beforeEach(() => { + resetSpies(); + }); + + describe('Gather Mode & Audit Mode', () => { + const url = 'https://example.com'; + const generateConfig = _ => new Config({ + passes: [{ + gatherers: ['viewport-dimensions'], + }], + audits: ['content-width'], + }); + + it('-G gathers, quits, and doesn\'t run audits', () => { + const opts = {url, config: generateConfig(), driverMock, flags: {gatherMode: true}}; + return Runner.run(null, opts).then(_ => { + assert.equal(loadArtifactsSpy.called, false, 'loadArtifacts was called'); + + assert.equal(saveArtifactsSpy.called, true, 'saveArtifacts was not called'); + const saveArtifactArg = saveArtifactsSpy.getCall(0).args[0]; + assert.ok(saveArtifactArg.ViewportDimensions); + assert.ok(saveArtifactArg.devtoolsLogs.defaultPass.length > 100); + + assert.equal(gatherRunnerRunSpy.called, true, 'GatherRunner.run was not called'); + assert.equal(runAuditSpy.called, false, '_runAudit was called'); + }); + }); + + // uses the files on disk from the -G test. ;) + it('-A audits from saved artifacts and doesn\'t gather', () => { + const opts = {url, config: generateConfig(), driverMock, flags: {auditMode: true}}; + return Runner.run(null, opts).then(_ => { + assert.equal(loadArtifactsSpy.called, true, 'loadArtifacts was not called'); + assert.equal(gatherRunnerRunSpy.called, false, 'GatherRunner.run was called'); + assert.equal(saveArtifactsSpy.called, false, 'saveArtifacts was called'); + assert.equal(runAuditSpy.called, true, '_runAudit was not called'); + }); + }); + + it('-GA is a normal run but it saves artifacts to disk', () => { + const opts = {url, config: generateConfig(), driverMock, + flags: {auditMode: true, gatherMode: true}}; + return Runner.run(null, opts).then(_ => { + assert.equal(loadArtifactsSpy.called, false, 'loadArtifacts was called'); + assert.equal(gatherRunnerRunSpy.called, true, 'GatherRunner.run was not called'); + assert.equal(saveArtifactsSpy.called, true, 'saveArtifacts was not called'); + assert.equal(runAuditSpy.called, true, '_runAudit was not called'); + }); + }); + + it('non -G/-A run doesn\'t save artifacts to disk', () => { + const opts = {url, config: generateConfig(), driverMock}; + return Runner.run(null, opts).then(_ => { + assert.equal(loadArtifactsSpy.called, false, 'loadArtifacts was called'); + assert.equal(gatherRunnerRunSpy.called, true, 'GatherRunner.run was not called'); + assert.equal(saveArtifactsSpy.called, false, 'saveArtifacts was called'); + assert.equal(runAuditSpy.called, true, '_runAudit was not called'); + }); + }); + }); + it('expands gatherers', () => { const url = 'https://example.com'; const config = new Config({ @@ -28,10 +105,12 @@ describe('Runner', () => { }); return Runner.run(null, {url, config, driverMock}).then(_ => { + assert.equal(gatherRunnerRunSpy.called, true, 'GatherRunner.run was not called'); assert.ok(typeof config.passes[0].gatherers[0] === 'object'); }); }); + it('rejects when given neither passes nor artifacts', () => { const url = 'https://example.com'; const config = new Config({ @@ -44,7 +123,7 @@ describe('Runner', () => { .then(_ => { assert.ok(false); }, err => { - assert.ok(/The config must provide passes/.test(err.message)); + assert.ok(/No browser artifacts are either/.test(err.message)); }); }); @@ -264,7 +343,7 @@ describe('Runner', () => { }); }); - it('rejects when given neither audits nor auditResults', () => { + it('rejects when not given audits to run (and not -G)', () => { const url = 'https://example.com'; const config = new Config({ passes: [{ @@ -276,47 +355,41 @@ describe('Runner', () => { .then(_ => { assert.ok(false); }, err => { - assert.ok(/The config must provide passes/.test(err.message)); + assert.ok(/No audits to evaluate/.test(err.message)); }); }); - it('accepts existing auditResults', () => { + it('returns data even if no config categories are provided', () => { const url = 'https://example.com'; const config = new Config({ - auditResults: [{ - name: 'content-width', - rawValue: true, - score: true, - displayValue: '', + passes: [{ + gatherers: ['viewport-dimensions'], }], - - categories: { - category: { - name: 'Category', - description: '', - audits: [ - {id: 'content-width', weight: 1}, - ], - }, - }, + audits: [ + 'content-width', + ], }); return Runner.run(null, {url, config, driverMock}).then(results => { - // Mostly checking that this did not throw, but check representative values. + assert.ok(results.lighthouseVersion); + assert.ok(results.generatedTime); assert.equal(results.initialUrl, url); - assert.strictEqual(results.audits['content-width'].rawValue, true); + assert.equal(gatherRunnerRunSpy.called, true, 'GatherRunner.run was not called'); + assert.equal(results.audits['content-width'].name, 'content-width'); + assert.equal(results.score, 0); }); }); + it('returns reportCategories', () => { const url = 'https://example.com'; const config = new Config({ - auditResults: [{ - name: 'content-width', - rawValue: true, - score: true, - displayValue: 'display', + passes: [{ + gatherers: ['viewport-dimensions'], }], + audits: [ + 'content-width', + ], categories: { category: { name: 'Category', @@ -332,14 +405,15 @@ describe('Runner', () => { assert.ok(results.lighthouseVersion); assert.ok(results.generatedTime); assert.equal(results.initialUrl, url); + assert.equal(gatherRunnerRunSpy.called, true, 'GatherRunner.run was not called'); assert.equal(results.audits['content-width'].name, 'content-width'); assert.equal(results.reportCategories[0].score, 100); assert.equal(results.reportCategories[0].audits[0].id, 'content-width'); assert.equal(results.reportCategories[0].audits[0].score, 100); - assert.equal(results.reportCategories[0].audits[0].result.displayValue, 'display'); }); }); + it('rejects when not given a URL', () => { return Runner.run({}, {}).then(_ => assert.ok(false), _ => assert.ok(true)); }); @@ -428,32 +502,6 @@ describe('Runner', () => { }); }); - it('results include artifacts when given auditResults', () => { - const url = 'https://example.com'; - const config = new Config({ - auditResults: [{ - name: 'is-on-https', - rawValue: true, - score: true, - displayValue: '', - }], - - artifacts: { - HTTPS: { - value: true, - }, - }, - }); - - return Runner.run(null, {url, config, driverMock}).then(results => { - assert.strictEqual(results.artifacts.HTTPS.value, true); - - for (const method of Object.keys(computedArtifacts)) { - assert.ok(results.artifacts.hasOwnProperty(method)); - } - }); - }); - it('includes any LighthouseRunWarnings from artifacts in output', () => { const url = 'https://example.com'; const LighthouseRunWarnings = [ diff --git a/lighthouse-core/test/scoring-test.js b/lighthouse-core/test/scoring-test.js index 09245934e687..89d585e39fb4 100644 --- a/lighthouse-core/test/scoring-test.js +++ b/lighthouse-core/test/scoring-test.js @@ -68,7 +68,7 @@ describe('ReportScoring', () => { }); it('should score the categories', () => { - const auditResults = { + const resultsByAuditId = { 'my-audit': {rawValue: 'you passed'}, 'my-boolean-audit': {score: true, extendedInfo: {}}, 'my-scored-audit': {score: 100}, @@ -88,7 +88,7 @@ describe('ReportScoring', () => { ], }, }, - }, auditResults); + }, resultsByAuditId); assert.equal(result.categories.length, 2); assert.equal(result.categories[0].score, 0); diff --git a/lighthouse-extension/gulpfile.js b/lighthouse-extension/gulpfile.js index 6eb0cb1e02d7..d4657c32d90f 100644 --- a/lighthouse-extension/gulpfile.js +++ b/lighthouse-extension/gulpfile.js @@ -125,6 +125,8 @@ gulp.task('browserify-lighthouse', () => { .ignore('url') .ignore('debug/node') .ignore('raven') + .ignore('mkdirp') + .ignore('rimraf') .ignore('pako/lib/zlib/inflate.js'); // Expose the audits, gatherers, and computed artifacts so they can be dynamically loaded. diff --git a/package.json b/package.json index dfc2ea019ffc..71e8c234346e 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "mocha": "^3.2.0", "npm-run-posix-or-windows": "^2.0.2", "typescript": "^2.6.1", + "sinon": "^2.3.5", "zone.js": "^0.7.3" }, "dependencies": { @@ -87,7 +88,6 @@ "inquirer": "^3.3.0", "jpeg-js": "0.1.2", "js-library-detector": "^4.3.1", - "json-stringify-safe": "5.0.1", "lighthouse-logger": "^1.0.0", "metaviewport-parser": "0.1.0", "mkdirp": "0.5.1", diff --git a/typings/externs.d.ts b/typings/externs.d.ts index 3b7b96b56f73..213f308eed48 100644 --- a/typings/externs.d.ts +++ b/typings/externs.d.ts @@ -12,7 +12,6 @@ export interface Flags { chromeFlags: string; output: any; outputPath: string; - saveArtifacts: boolean; saveAssets: boolean; view: boolean; maxWaitForLoad: number; @@ -22,6 +21,8 @@ export interface Flags { enableErrorReporting: boolean; listAllAudits: boolean; listTraceCategories: boolean; + auditMode: boolean; + gatherMode: boolean; configPath?: string; perf: boolean; verbose: boolean; diff --git a/yarn.lock b/yarn.lock index 4f13c9564ab8..cc7b71ae02fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1204,6 +1204,10 @@ diff@1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/diff/-/diff-1.4.0.tgz#7f28d2eb9ee7b15a97efd89ce63dcfdaa3ccbabf" +diff@^3.1.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-3.4.0.tgz#b1d85507daf3964828de54b37d0d73ba67dda56c" + doctrine@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.0.0.tgz#c73d8d2909d22291e1a007a395804da8b665fe63" @@ -1601,6 +1605,12 @@ form-data@~2.1.1: combined-stream "^1.0.5" mime-types "^2.1.12" +formatio@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/formatio/-/formatio-1.2.0.tgz#f3b2167d9068c4698a8d51f4f760a39a54d818eb" + dependencies: + samsam "1.x" + fs-exists-sync@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz#982d6893af918e72d08dec9e8673ff2b5a8d6add" @@ -2455,7 +2465,7 @@ json-stable-stringify@^1.0.1: dependencies: jsonify "~0.0.0" -json-stringify-safe@5.0.1, json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" @@ -2749,6 +2759,10 @@ log-driver@1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/log-driver/-/log-driver-1.2.5.tgz#7ae4ec257302fd790d557cb10c97100d857b0056" +lolex@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/lolex/-/lolex-1.6.0.tgz#3a9a0283452a47d7439e72731b9e07d7386e49f6" + longest@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" @@ -2935,6 +2949,10 @@ mute-stream@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab" +native-promise-only@^0.8.1: + version "0.8.1" + resolved "https://registry.yarnpkg.com/native-promise-only/-/native-promise-only-0.8.1.tgz#20a318c30cb45f71fe7adfbf7b21c99c1472ef11" + natives@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/natives/-/natives-1.1.0.tgz#e9ff841418a6b2ec7a495e939984f78f163e6e31" @@ -3192,6 +3210,12 @@ path-root@^0.1.1: dependencies: path-root-regex "^0.1.0" +path-to-regexp@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.7.0.tgz#59fde0f435badacba103a84e9d3bc64e96b9937d" + dependencies: + isarray "0.0.1" + path-type@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -3602,6 +3626,10 @@ safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" +samsam@1.x, samsam@^1.1.3: + version "1.3.0" + resolved "https://registry.yarnpkg.com/samsam/-/samsam-1.3.0.tgz#8d1d9350e25622da30de3e44ba692b5221ab7c50" + sax@^1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.2.tgz#fd8631a23bc7826bef5d871bdb87378c95647828" @@ -3658,6 +3686,19 @@ signal-exit@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" +sinon@^2.3.5: + version "2.4.1" + resolved "https://registry.yarnpkg.com/sinon/-/sinon-2.4.1.tgz#021fd64b54cb77d9d2fb0d43cdedfae7629c3a36" + dependencies: + diff "^3.1.0" + formatio "1.2.0" + lolex "^1.6.0" + native-promise-only "^0.8.1" + path-to-regexp "^1.7.0" + samsam "^1.1.3" + text-encoding "0.6.4" + type-detect "^4.0.0" + slash@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" @@ -3914,6 +3955,10 @@ term-size@^0.1.0: dependencies: execa "^0.4.0" +text-encoding@0.6.4: + version "0.6.4" + resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.6.4.tgz#e399a982257a276dae428bb92845cb71bdc26d19" + text-extensions@^1.0.0: version "1.7.0" resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.7.0.tgz#faaaba2625ed746d568a23e4d0aacd9bf08a8b39" @@ -4023,13 +4068,17 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" +type-detect@^4.0.0: + version "4.0.3" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.3.tgz#0e3f2670b44099b0b46c284d136a7ef49c74c2ea" + typedarray@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" typescript@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.6.1.tgz#ef39cdea27abac0b500242d6726ab90e0c846631" + version "2.6.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.6.2.tgz#3c5b6fd7f6de0914269027f03c0946758f7673a4" uglify-js@^2.6: version "2.7.3"