From 50f7a9c51fb76d156c4495bffb5f4322705d558d Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Tue, 12 Feb 2019 11:03:39 -0800 Subject: [PATCH 1/6] smokehouse: count failedUrl/errorCode mismatches (#7225) --- lighthouse-cli/test/smokehouse/smokehouse.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index 4a304496e891..3b5ea793737b 100755 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -311,11 +311,23 @@ function reportAssertion(assertion) { * @return {{passed: number, failed: number}} */ function report(results) { + let correctCount = 0; + let failedCount = 0; + reportAssertion(results.finalUrl); + if (results.finalUrl.equal) { + correctCount++; + } else { + failedCount++; + } + reportAssertion(results.errorCode); + if (results.errorCode.equal) { + correctCount++; + } else { + failedCount++; + } - let correctCount = 0; - let failedCount = 0; results.audits.forEach(auditAssertion => { if (auditAssertion.equal) { correctCount++; From c377eb05f769cf38be1e310a191388abffb99034 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Tue, 12 Feb 2019 11:40:09 -0800 Subject: [PATCH 2/6] show all assertions regardless of pass/fail state --- lighthouse-cli/test/smokehouse/smokehouse.js | 26 ++++++-------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index 3b5ea793737b..78cdaf3b4d36 100755 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -276,8 +276,12 @@ function reportAssertion(assertion) { RegExp.prototype.toJSON = RegExp.prototype.toString; if (assertion.equal) { - console.log(` ${log.greenify(log.tick)} ${assertion.category}: ` + - log.greenify(assertion.actual)); + if (assertion.actual !== null && typeof assertion.actual === 'object') { + console.log(` ${log.greenify(log.tick)} ${assertion.category}`); + } else { + console.log(` ${log.greenify(log.tick)} ${assertion.category}: ` + + log.greenify(assertion.actual)); + } } else { if (assertion.diff) { const diff = assertion.diff; @@ -314,26 +318,12 @@ function report(results) { let correctCount = 0; let failedCount = 0; - reportAssertion(results.finalUrl); - if (results.finalUrl.equal) { - correctCount++; - } else { - failedCount++; - } - - reportAssertion(results.errorCode); - if (results.errorCode.equal) { - correctCount++; - } else { - failedCount++; - } - - results.audits.forEach(auditAssertion => { + [results.finalUrl, results.errorCode, ...results.audits].forEach(auditAssertion => { + reportAssertion(auditAssertion); if (auditAssertion.equal) { correctCount++; } else { failedCount++; - reportAssertion(auditAssertion); } }); From 97ab34dace84a51ecb2e367acae325bd03ef9288 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Tue, 12 Feb 2019 14:55:21 -0800 Subject: [PATCH 3/6] verbose env flag --- lighthouse-cli/test/smokehouse/smokehouse.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index 78cdaf3b4d36..2ebd1e2ef6b5 100755 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -35,6 +35,7 @@ const PAGE_HUNG_EXIT_CODE = 68; const INSECURE_DOCUMENT_REQUEST_EXIT_CODE = 69; const RETRIES = 3; const NUMERICAL_EXPECTATION_REGEXP = /^(<=?|>=?)((\d|\.)+)$/; +const VERBOSE = process.env.LH_SMOKE_VERBOSE === '1'; /** * Attempt to resolve a path locally. If this fails, attempts to locate the path @@ -319,12 +320,15 @@ function report(results) { let failedCount = 0; [results.finalUrl, results.errorCode, ...results.audits].forEach(auditAssertion => { - reportAssertion(auditAssertion); if (auditAssertion.equal) { correctCount++; } else { failedCount++; } + + if (!auditAssertion.equal || VERBOSE) { + reportAssertion(auditAssertion); + } }); const plural = correctCount === 1 ? '' : 's'; From 255bfbc2659385af7db88ede01718bda4bc19d15 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 13 Feb 2019 08:55:17 -0800 Subject: [PATCH 4/6] rename debug env, use Boolean --- lighthouse-cli/test/smokehouse/smokehouse.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index 2ebd1e2ef6b5..b57d7ef73d53 100755 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -35,7 +35,7 @@ const PAGE_HUNG_EXIT_CODE = 68; const INSECURE_DOCUMENT_REQUEST_EXIT_CODE = 69; const RETRIES = 3; const NUMERICAL_EXPECTATION_REGEXP = /^(<=?|>=?)((\d|\.)+)$/; -const VERBOSE = process.env.LH_SMOKE_VERBOSE === '1'; +const VERBOSE = Boolean(process.env.LH_SMOKE_VERBOSE); /** * Attempt to resolve a path locally. If this fails, attempts to locate the path @@ -63,7 +63,7 @@ function resolveLocalOrCwd(payloadPath) { * @return {ExpectedLHR} */ function runLighthouse(url, configPath, isDebug) { - isDebug = isDebug || Boolean(process.env.SMOKEHOUSE_DEBUG); + isDebug = isDebug || Boolean(process.env.LH_SMOKE_DEBUG); const command = 'node'; const outputPath = `smokehouse-${Math.round(Math.random() * 100000)}.report.json`; From 72b22197b09f9146c6ce730cc5e5f5675a8f606b Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 13 Feb 2019 10:20:44 -0800 Subject: [PATCH 5/6] isPlainObject --- lighthouse-cli/test/smokehouse/smokehouse.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index b57d7ef73d53..926ad90ef7dc 100755 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -265,6 +265,13 @@ function collateResults(actual, expected) { }; } +/** + * @param {*} obj + */ +function isPlainObject(obj) { + return Object.prototype.toString.call(obj) === '[object Object]'; +} + /** * Log the result of an assertion of actual and expected results. * @param {Comparison} assertion @@ -277,7 +284,7 @@ function reportAssertion(assertion) { RegExp.prototype.toJSON = RegExp.prototype.toString; if (assertion.equal) { - if (assertion.actual !== null && typeof assertion.actual === 'object') { + if (isPlainObject(assertion.actual)) { console.log(` ${log.greenify(log.tick)} ${assertion.category}`); } else { console.log(` ${log.greenify(log.tick)} ${assertion.category}: ` + From ab4eb4061a5446d4a2f880ea3342676ba03fcd18 Mon Sep 17 00:00:00 2001 From: Brendan Kenny Date: Thu, 14 Feb 2019 10:38:02 -0800 Subject: [PATCH 6/6] Update lighthouse-cli/test/smokehouse/smokehouse.js Co-Authored-By: Hoten --- lighthouse-cli/test/smokehouse/smokehouse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index 926ad90ef7dc..d7f5ff316604 100755 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -266,7 +266,7 @@ function collateResults(actual, expected) { } /** - * @param {*} obj + * @param {unknown} obj */ function isPlainObject(obj) { return Object.prototype.toString.call(obj) === '[object Object]';