test(smokehouse): add numeric comparisons (#2356)

GoogleChrome · May 27, 2017 · d5854b6 · d5854b6
1 parent 5107ab2
commit d5854b6
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 14 deletions.
diff --git a/lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js b/lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js
@@ -36,8 +36,7 @@ module.exports = [
       //   }
       // },
       'offscreen-images': {
-        // TODO: re-enable score assertions when we have more flexible assertions like < 100
-        // score: 65,
+        score: '<100',
         extendedInfo: {
           value: {
             results: {
@@ -47,7 +46,7 @@ module.exports = [
         }
       },
       'uses-webp-images': {
-        // score: 65,
+        score: '<100',
         extendedInfo: {
           value: {
             results: {
@@ -57,7 +56,7 @@ module.exports = [
         }
       },
       'uses-optimized-images': {
-        // score: 65,
+        score: '<100',
         extendedInfo: {
           value: {
             results: {
@@ -67,7 +66,7 @@ module.exports = [
         }
       },
       'uses-responsive-images': {
-        // score: 90,
+        score: '<100',
         extendedInfo: {
           value: {
             results: {

diff --git a/lighthouse-cli/test/smokehouse/perf/expectations.js b/lighthouse-cli/test/smokehouse/perf/expectations.js
@@ -25,7 +25,7 @@ module.exports = [
     url: 'http://localhost:10200/online-only.html',
     audits: {
       'speed-index-metric': {
-        score: 100,
+        score: '>=90',
         extendedInfo: {
           value: {
             timings: {},
@@ -34,9 +34,15 @@ module.exports = [
           }
         }
       },
+      'first-meaningful-paint': {
+        score: '>=90',
+      },
       'first-interactive': {
-        score: 100,
-      }
+        score: '>=90',
+      },
+      'consistently-interactive': {
+        score: '>=90',
+      },
     }
   },
 ];
diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js
@@ -29,6 +29,7 @@ const DEFAULT_EXPECTATIONS_PATH = 'pwa-expectations';
 
 const PROTOCOL_TIMEOUT_EXIT_CODE = 67;
 const RETRIES = 3;
+const NUMERICAL_EXPECTATION_REGEXP = /^(<=?|>=?)(\d+)$/;
 
 
 /**
@@ -104,6 +105,39 @@ function runLighthouse(url, configPath, saveAssetsPath) {
   return JSON.parse(runResults.stdout);
 }
 
+/**
+ * Checks if the actual value matches the expectation. Does not recursively search. This supports
+ *    - Greater than/less than operators, e.g. "<100", ">90"
+ *    - Regular expressions
+ *    - Strict equality
+ *
+ * @param {*} actual
+ * @param {*} expected
+ * @return {boolean}
+ */
+function matchesExpectation(actual, expected) {
+  if (typeof actual === 'number' && NUMERICAL_EXPECTATION_REGEXP.test(expected)) {
+    const parts = expected.match(NUMERICAL_EXPECTATION_REGEXP);
+    const operator = parts[1];
+    const number = parseInt(parts[2]);
+    switch (operator) {
+      case '>':
+        return actual > number;
+      case '>=':
+        return actual >= number;
+      case '<':
+        return actual < number;
+      case '<=':
+        return actual <= number;
+    }
+  } else if (typeof actual === 'string' && expected instanceof RegExp && expected.test(actual)) {
+    return true;
+  } else {
+    // Strict equality check, plus NaN equivalence.
+    return Object.is(actual, expected);
+  }
+}
+
 /**
  * Walk down expected result, comparing to actual result. If a difference is found,
  * the path to the difference is returned, along with the expected primitive value
@@ -118,12 +152,7 @@ function runLighthouse(url, configPath, saveAssetsPath) {
  * @return {({path: string, actual: *, expected: *}|null)}
  */
 function findDifference(path, actual, expected) {
-  // Strict equality check, plus NaN equivalence.
-  if (Object.is(actual, expected)) {
-    return null;
-  }
-
-  if (typeof actual === 'string' && expected instanceof RegExp && expected.test(actual)) {
+  if (matchesExpectation(actual, expected)) {
     return null;
   }