feat: support Config category and audit whitelisting (#1988)

lighthouse-core/config/config.js

@@ -272,6 +272,15 @@ class Config {
       configJSON = Config.extendConfigJSON(deepClone(defaultConfig), configJSON);
     }
 
+    // Generate a limited config if specified
+    if (configJSON.settings &&
+        (Array.isArray(configJSON.settings.onlyCategories) ||
+        Array.isArray(configJSON.settings.onlyAudits))) {
+      const categoryIds = configJSON.settings.onlyCategories;
+      const auditIds = configJSON.settings.onlyAudits;
+      configJSON = Config.generateNewFilteredConfig(configJSON, categoryIds, auditIds);
+    }
+
     // Store the directory of the config path, if one was provided.
     this._configDir = configPath ? path.dirname(configPath) : undefined;
 
@@ -316,14 +325,15 @@ class Config {
   /**
    * Filter out any unrequested items from the config, based on requested top-level categories.
    * @param {!Object} oldConfig Lighthouse config object
-   * @param {!Array<string>} categoryIds ID values of categories to include
+   * @param {!Array<string>=} categoryIds ID values of categories to include
+   * @param {!Array<string>=} auditIds ID values of categories to include
    * @return {!Object} A new config
    */
-  static generateNewConfigOfCategories(oldConfig, categoryIds) {
+  static generateNewFilteredConfig(oldConfig, categoryIds, auditIds) {
     // 0. Clone config to avoid mutating it
-    const config = JSON.parse(JSON.stringify(oldConfig));
+    const config = deepClone(oldConfig);
     // 1. Filter to just the chosen categories
-    config.categories = Config.filterCategories(config.categories, categoryIds);
+    config.categories = Config.filterCategoriesAndAudits(config.categories, categoryIds, auditIds);
 
     // 2. Resolve which audits will need to run
     const requestedAuditNames = Config.getAuditIdsInCategories(config.categories);
@@ -341,17 +351,48 @@ class Config {
   }
 
   /**
-   * Filter out any unrequested categories from the categories object.
+   * Filter out any unrequested categories or audits from the categories object.
    * @param {!Object<string, {audits: !Array<{id: string}>}>} categories
-   * @param {Array<string>=} categoryIds
+   * @param {!Array<string>=} categoryIds
+   * @param {!Array<string>=} auditIds
    * @return {!Object<string, {audits: !Array<{id: string}>}>}
    */
-  static filterCategories(oldCategories, categoryIds = []) {
+  static filterCategoriesAndAudits(oldCategories, categoryIds = [], auditIds = []) {
     const categories = {};
 
+    // warn if the category is not found
+    categoryIds.forEach(categoryId => {
+      if (!oldCategories[categoryId]) {
+        log.warn('config', `unrecognized category in 'onlyCategories': ${categoryId}`);
+      }
+    });
+
+    // warn if the audit is not found in a category
+    auditIds.forEach(auditId => {
+      const foundCategory = Object.keys(oldCategories).find(categoryId => {
+        const audits = oldCategories[categoryId].audits;
+        return audits.find(candidate => candidate.id === auditId);
+      });
+
+      if (!foundCategory) {
+        log.warn('config', `unrecognized audit in 'onlyAudits': ${auditId}`);
+      }
+
+      if (categoryIds.includes(foundCategory)) {
+        log.warn('config', `${auditId} in 'onlyAudits' is already included by ` +
+            `${foundCategory} in 'onlyCategories'`);
+      }
+    });
+
     Object.keys(oldCategories).forEach(categoryId => {
       if (categoryIds.includes(categoryId)) {
         categories[categoryId] = oldCategories[categoryId];
+      } else {
+        const newCategory = deepClone(oldCategories[categoryId]);
+        newCategory.audits = newCategory.audits.filter(audit => auditIds.includes(audit.id));
+        if (newCategory.audits.length) {
+          categories[categoryId] = newCategory;
+        }
       }
     });
 

lighthouse-core/config/default.js

@@ -1,5 +1,6 @@
 /* eslint-disable */
 module.exports = {
+  "settings": {},
   "passes": [{
     "passName": "defaultPass",
     "recordNetwork": true,

lighthouse-core/config/perf.json

@@ -1,114 +1,6 @@
 {
-  "passes": [{
-    "recordNetwork": true,
-    "recordTrace": true,
-    "pauseBeforeTraceEndMs": 5000,
-    "useThrottling": true,
-    "gatherers": [
-      "image-usage",
-      "viewport-dimensions",
-      "dobetterweb/domstats",
-      "dobetterweb/tags-blocking-first-paint",
-      "dobetterweb/optimized-images"
-    ]
+  "extends": "lighthouse:default",
+  "settings": {
+    "onlyCategories": ["performance"]
   }
-  ],
-
-  "audits": [
-    "first-meaningful-paint",
-    "speed-index-metric",
-    "estimated-input-latency",
-    "time-to-interactive",
-    "user-timings",
-    "critical-request-chains",
-    "byte-efficiency/uses-optimized-images",
-    "byte-efficiency/uses-responsive-images",
-    "dobetterweb/dom-size",
-    "dobetterweb/link-blocking-first-paint",
-    "dobetterweb/script-blocking-first-paint"
-  ],
-
-  "categories": {
-    "performance": {
-      "name": "Performance",
-      "description": "These encapsulate your app's performance.",
-      "audits": [
-        {"id": "first-meaningful-paint", "weight": 5},
-        {"id": "speed-index-metric", "weight": 1},
-        {"id": "estimated-input-latency", "weight": 1},
-        {"id": "time-to-interactive", "weight": 5},
-        {"id": "link-blocking-first-paint", "weight": 0},
-        {"id": "script-blocking-first-paint", "weight": 0},
-        {"id": "uses-optimized-images", "weight": 0},
-        {"id": "uses-responsive-images", "weight": 0},
-        {"id": "dom-size", "weight": 0},
-        {"id": "critical-request-chains", "weight": 0},
-        {"id": "user-timings", "weight": 0}
-      ]
-    }
-  },
-
-  "aggregations": [{
-    "name": "Performance metrics",
-    "description": "",
-    "scored": false,
-    "categorizable": false,
-    "items": [{
-      "audits": {
-        "first-meaningful-paint": {
-          "expectedValue": 100,
-          "weight": 1
-        },
-        "speed-index-metric": {
-          "expectedValue": 100,
-          "weight": 1
-        },
-        "estimated-input-latency": {
-          "expectedValue": 100,
-          "weight": 1
-        },
-        "time-to-interactive": {
-          "expectedValue": 100,
-          "weight": 1
-        }
-      }
-    }]
-  },{
-    "name": "Performance diagnostics",
-    "description": "",
-    "scored": false,
-    "categorizable": false,
-    "items": [{
-      "audits": {
-        "uses-optimized-images": {
-          "expectedValue": true,
-          "weight": 1
-        },
-        "uses-responsive-images": {
-          "expectedValue": true,
-          "weight": 1
-        },
-        "critical-request-chains": {
-          "expectedValue": true,
-          "weight": 1
-        },
-        "link-blocking-first-paint": {
-          "expectedValue": true,
-          "weight": 1
-        },
-        "script-blocking-first-paint": {
-          "expectedValue": true,
-          "weight": 1
-        },
-        "dom-size": {
-          "expectedValue": 100,
-          "weight": 1
-        },
-        "user-timings": {
-          "expectedValue": true,
-          "weight": 1
-        }
-      }
-    }]
-  }]
 }

lighthouse-core/test/config/config-test.js

@@ -265,6 +265,81 @@ describe('Config', () => {
     }), /meta.requiredArtifacts property/);
   });
 
+  it('filters the config', () => {
+    const config = new Config({
+      settings: {
+        onlyCategories: ['needed-category'],
+        onlyAudits: ['color-contrast'],
+      },
+      passes: [
+        {recordTrace: true, gatherers: []},
+        {recordNetwork: true, gatherers: ['accessibility']},
+      ],
+      audits: [
+        'accessibility/color-contrast',
+        'first-meaningful-paint',
+        'time-to-interactive',
+        'estimated-input-latency',
+      ],
+      categories: {
+        'needed-category': {
+          audits: [
+            {id: 'first-meaningful-paint'},
+            {id: 'time-to-interactive'},
+          ],
+        },
+        'other-category': {
+          audits: [
+            {id: 'color-contrast'},
+            {id: 'estimated-input-latency'},
+          ],
+        },
+        'unused-category': {
+          audits: [
+            {id: 'estimated-input-latency'},
+          ]
+        }
+      },
+    });
+
+    assert.ok(config.audits.length, 3);
+    assert.equal(config.passes.length, 2);
+    assert.ok(!config.categories['unused-category'], 'removes unused categories');
+    assert.equal(config.categories['needed-category'].audits.length, 2);
+    assert.equal(config.categories['other-category'].audits.length, 1);
+  });
+
+  it('filtering works with extension', () => {
+    const config = new Config({
+      extends: true,
+      settings: {
+        onlyCategories: ['performance'],
+        onlyAudits: ['is-on-https'],
+      },
+    });
+
+    assert.ok(config.audits.length, 'inherited audits by extension');
+    assert.equal(config.audits.length, origConfig.categories.performance.audits.length + 1);
+    assert.equal(config.passes.length, 2, 'filtered out passes');
+  });
+
+  it('warns for invalid filters', () => {
+    const warnings = [];
+    const saveWarning = evt => warnings.push(evt);
+    log.events.addListener('warning', saveWarning);
+    const config = new Config({
+      extends: true,
+      settings: {
+        onlyCategories: ['performance', 'missing-category'],
+        onlyAudits: ['time-to-interactive', 'missing-audit'],
+      },
+    });
+
+    log.events.removeListener('warning', saveWarning);
+    assert.ok(config, 'failed to generate config');
+    assert.equal(warnings.length, 3, 'did not warn enough');
+  });
+
   describe('artifact loading', () => {
     it('expands artifacts', () => {
       const config = new Config({
@@ -384,39 +459,61 @@ describe('Config', () => {
   describe('generateConfigOfCategories', () => {
     it('should not mutate the original config', () => {
       const configCopy = JSON.parse(JSON.stringify(origConfig));
-      Config.generateNewConfigOfCategories(configCopy, ['performance']);
+      Config.generateNewFilteredConfig(configCopy, ['performance']);
       assert.deepStrictEqual(configCopy, origConfig, 'no mutations');
     });
 
     it('should filter out other passes if passed Performance', () => {
       const totalAuditCount = origConfig.audits.length;
-      const config = Config.generateNewConfigOfCategories(origConfig, ['performance']);
+      const config = Config.generateNewFilteredConfig(origConfig, ['performance']);
       assert.equal(Object.keys(config.categories).length, 1, 'other categories are present');
       assert.equal(config.passes.length, 2, 'incorrect # of passes');
       assert.ok(config.audits.length < totalAuditCount, 'audit filtering probably failed');
     });
 
     it('should filter out other passes if passed PWA', () => {
       const totalAuditCount = origConfig.audits.length;
-      const config = Config.generateNewConfigOfCategories(origConfig, ['pwa']);
+      const config = Config.generateNewFilteredConfig(origConfig, ['pwa']);
       assert.equal(Object.keys(config.categories).length, 1, 'other categories are present');
       assert.ok(config.audits.length < totalAuditCount, 'audit filtering probably failed');
     });
 
     it('should filter out other passes if passed Best Practices', () => {
       const totalAuditCount = origConfig.audits.length;
-      const config = Config.generateNewConfigOfCategories(origConfig, ['best-practices']);
+      const config = Config.generateNewFilteredConfig(origConfig, ['best-practices']);
       assert.equal(Object.keys(config.categories).length, 1, 'other categories are present');
       assert.equal(config.passes.length, 2, 'incorrect # of passes');
       assert.ok(config.audits.length < totalAuditCount, 'audit filtering probably failed');
     });
 
     it('should only run audits for ones named by the category', () => {
-      const config = Config.generateNewConfigOfCategories(origConfig, ['performance']);
+      const config = Config.generateNewFilteredConfig(origConfig, ['performance']);
       const selectedCategory = origConfig.categories.performance;
       const auditCount = Object.keys(selectedCategory.audits).length;
 
       assert.equal(config.audits.length, auditCount, '# of audits match aggregation list');
     });
+
+    it('should only run specified audits', () => {
+      const config = Config.generateNewFilteredConfig(origConfig, [], ['works-offline']);
+      assert.equal(config.passes.length, 2, 'incorrect # of passes');
+      assert.equal(config.audits.length, 1, 'audit filtering failed');
+    });
+
+    it('should combine audits and categories additively', () => {
+      const config = Config.generateNewFilteredConfig(origConfig, ['performance'], ['is-on-https']);
+      const selectedCategory = origConfig.categories.performance;
+      const auditCount = Object.keys(selectedCategory.audits).length + 1;
+      assert.equal(config.passes.length, 2, 'incorrect # of passes');
+      assert.equal(config.audits.length, auditCount, 'audit filtering failed');
+    });
+
+    it('should support redundant filtering', () => {
+      const config = Config.generateNewFilteredConfig(origConfig, ['pwa'], ['is-on-https']);
+      const selectedCategory = origConfig.categories.pwa;
+      const auditCount = Object.keys(selectedCategory.audits).length;
+      assert.equal(config.passes.length, 3, 'incorrect # of passes');
+      assert.equal(config.audits.length, auditCount, 'audit filtering failed');
+    });
   });
 });

lighthouse-extension/app/src/lighthouse-background.js

@@ -99,8 +99,10 @@ function filterOutArtifacts(result) {
  * @return {!Promise}
  */
 window.runLighthouseForConnection = function(connection, url, options, categoryIDs) {
-  const newConfig = Config.generateNewConfigOfCategories(defaultConfig, categoryIDs);
-  const config = new Config(newConfig);
+  const config = new Config({
+    extends: 'lighthouse:default',
+    settings: {onlyCategories: categoryIDs},
+  });
 
   // Add url and config to fresh options object.
   const runOptions = Object.assign({}, options, {url, config});