core(scoring): tweak scoring thresholds based on HTTPArchive data (Go…

…ogleChrome#5022)
kdzwinel · Aug 16, 2018 · ce60ab8 · ce60ab8
1 parent 36bbe55
commit ce60ab8
Show file tree

Hide file tree

Showing 12 changed files with 47 additions and 33 deletions.
diff --git a/lighthouse-core/audits/byte-efficiency/uses-long-cache-ttl.js b/lighthouse-core/audits/byte-efficiency/uses-long-cache-ttl.js
@@ -37,9 +37,11 @@ class CacheHeaders extends Audit {
    */
   static get defaultOptions() {
     return {
-      // see https://www.desmos.com/calculator/zokzso8umm
+      // 50th and 75th percentiles HTTPArchive -> 50 and 75
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      // see https://www.desmos.com/calculator/8meohdnjbl
       scorePODR: 4 * 1024,
-      scoreMedian: 768 * 1024,
+      scoreMedian: 128 * 1024,
     };
   }
 

diff --git a/lighthouse-core/audits/consistently-interactive.js b/lighthouse-core/audits/consistently-interactive.js
@@ -35,9 +35,11 @@ class ConsistentlyInteractiveMetric extends Audit {
    */
   static get defaultOptions() {
     return {
-      // see https://www.desmos.com/calculator/uti67afozh
-      scorePODR: 1700,
-      scoreMedian: 10000,
+      // 75th and 90th percentiles HTTPArchive -> 50 and 75
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      // see https://www.desmos.com/calculator/dohd3b0sbr
+      scorePODR: 1200,
+      scoreMedian: 7300,
     };
   }
 

diff --git a/lighthouse-core/audits/dobetterweb/dom-size.js b/lighthouse-core/audits/dobetterweb/dom-size.js
@@ -48,9 +48,11 @@ class DOMSize extends Audit {
    */
   static get defaultOptions() {
     return {
-      // see https://www.desmos.com/calculator/9cyxpm5qgp
-      scorePODR: 2400,
-      scoreMedian: 3000,
+      // 25th and 50th percentiles HTTPArchive -> 50 and 75
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      // see https://www.desmos.com/calculator/vqot3wci4g
+      scorePODR: 700,
+      scoreMedian: 1400,
     };
   }
 

diff --git a/lighthouse-core/audits/first-contentful-paint.js b/lighthouse-core/audits/first-contentful-paint.js
@@ -28,8 +28,10 @@ class FirstContentfulPaint extends Audit {
    */
   static get defaultOptions() {
     return {
-      // see https://www.desmos.com/calculator/joz3pqttdq
-      scorePODR: 1600,
+      // 75th and 90th percentiles HTTPArchive -> 50 and 75
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      // see https://www.desmos.com/calculator/trv2goqvsd
+      scorePODR: 1400,
       scoreMedian: 4000,
     };
   }

diff --git a/lighthouse-core/audits/first-cpu-idle.js b/lighthouse-core/audits/first-cpu-idle.js
@@ -29,9 +29,11 @@ class FirstCPUIdle extends Audit {
    */
   static get defaultOptions() {
     return {
-      // see https://www.desmos.com/calculator/uti67afozh
-      scorePODR: 1700,
-      scoreMedian: 10000,
+      // 75th and 90th percentiles HTTPArchive -> 50 and 75
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      // see https://www.desmos.com/calculator/cwuavnclzj
+      scorePODR: 1400,
+      scoreMedian: 6500,
     };
   }
 

diff --git a/lighthouse-core/audits/first-meaningful-paint.js b/lighthouse-core/audits/first-meaningful-paint.js
@@ -28,8 +28,10 @@ class FirstMeaningfulPaint extends Audit {
    */
   static get defaultOptions() {
     return {
-      // see https://www.desmos.com/calculator/joz3pqttdq
-      scorePODR: 1600,
+      // 75th and 90th percentiles HTTPArchive -> 50 and 75
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      // see https://www.desmos.com/calculator/trv2goqvsd
+      scorePODR: 1400,
       scoreMedian: 4000,
     };
   }

diff --git a/lighthouse-core/audits/speed-index.js b/lighthouse-core/audits/speed-index.js
@@ -28,9 +28,11 @@ class SpeedIndex extends Audit {
    */
   static get defaultOptions() {
     return {
-      // see https://www.desmos.com/calculator/mdgjzchijg
-      scorePODR: 1250,
-      scoreMedian: 5500,
+      // 75th and 90th percentiles HTTPArchive -> 50 and 75
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      // see https://www.desmos.com/calculator/y1bg8ij7ti
+      scorePODR: 1700,
+      scoreMedian: 5800,
     };
   }
 

diff --git a/lighthouse-core/test/audits/byte-efficiency/uses-long-cache-ttl-test.js b/lighthouse-core/test/audits/byte-efficiency/uses-long-cache-ttl-test.js
@@ -166,7 +166,7 @@ describe('Cache headers audit', () => {
   it('ignores potentially uncacheable records', () => {
     networkRecords = [
       networkRecord({statusCode: 500}),
-      networkRecord({url: 'https://example.com/dynamic.js?userId=crazy'}),
+      networkRecord({url: 'https://example.com/dynamic.js?userId=crazy', transferSize: 10}),
       networkRecord({url: 'data:image/jpeg;base64,what'}),
       networkRecord({resourceType: WebInspector.resourceTypes.XHR}),
     ];

diff --git a/lighthouse-core/test/audits/consistently-interactive-test.js b/lighthouse-core/test/audits/consistently-interactive-test.js
@@ -31,7 +31,7 @@ describe('Performance: consistently-interactive audit', () => {
 
     const settings = {throttlingMethod: 'provided'};
     return ConsistentlyInteractive.audit(artifacts, {options, settings}).then(output => {
-      assert.equal(output.score, 0.99);
+      assert.equal(output.score, 0.97);
       assert.equal(Math.round(output.rawValue), 1582);
       assert.equal(output.displayValue, '1,580\xa0ms');
     });
@@ -49,7 +49,7 @@ describe('Performance: consistently-interactive audit', () => {
 
     const settings = {throttlingMethod: 'provided'};
     return ConsistentlyInteractive.audit(artifacts, {options, settings}).then(output => {
-      assert.equal(output.score, 0.95);
+      assert.equal(output.score, 0.89);
       assert.equal(Math.round(output.rawValue), 2712);
       assert.equal(output.displayValue, '2,710\xa0ms');
     });

diff --git a/lighthouse-core/test/audits/dobetterweb/dom-size-test.js b/lighthouse-core/test/audits/dobetterweb/dom-size-test.js
@@ -21,19 +21,19 @@ describe('Num DOM nodes audit', () => {
     },
   };
 
-  it('calculates score hitting top of distribution', () => {
+  it('calculates score hitting mid distribution', () => {
     const auditResult = DOMSize.audit(artifact, {options});
-    assert.equal(auditResult.score, 1);
+    assert.equal(auditResult.score, 0.43);
     assert.equal(auditResult.rawValue, numNodes);
     assert.equal(auditResult.displayValue, `${numNodes.toLocaleString()} nodes`);
     assert.equal(auditResult.details.items[0].totalNodes, numNodes.toLocaleString());
     assert.equal(auditResult.details.items[0].depth, '1');
     assert.equal(auditResult.details.items[0].width, '2');
   });
 
-  it('calculates score hitting mid distribution', () => {
-    artifact.DOMStats.totalDOMNodes = 3100;
-    assert.equal(DOMSize.audit(artifact, {options}).score, 0.43);
+  it('calculates score hitting top distribution', () => {
+    artifact.DOMStats.totalDOMNodes = 400;
+    assert.equal(DOMSize.audit(artifact, {options}).score, 1);
   });
 
   it('calculates score hitting bottom of distribution', () => {

diff --git a/lighthouse-core/test/audits/first-meaningful-paint-test.js b/lighthouse-core/test/audits/first-meaningful-paint-test.js
@@ -38,7 +38,7 @@ describe('Performance: first-meaningful-paint audit', () => {
     const context = {options, settings: {throttlingMethod: 'simulate'}};
     const fmpResult = await FMPAudit.audit(artifacts, context);
 
-    assert.equal(fmpResult.score, 0.75);
+    assert.equal(fmpResult.score, 0.73);
     assert.equal(fmpResult.displayValue, '2,850\xa0ms');
     assert.equal(Math.round(fmpResult.rawValue), 2851);
   });

diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
@@ -112,7 +112,7 @@
       "helpText": "A fast page load over a 3G network ensures a good mobile user experience. [Learn more](https://developers.google.com/web/tools/lighthouse/audits/fast-3g)."
     },
     "speed-index": {
-      "score": 0.62,
+      "score": 0.67,
       "displayValue": "4,420 ms",
       "rawValue": 4417,
       "scoreDisplayMode": "numeric",
@@ -267,7 +267,7 @@
       }
     },
     "first-cpu-idle": {
-      "score": 0.82,
+      "score": 0.65,
       "displayValue": "4,930 ms",
       "rawValue": 4927.278,
       "scoreDisplayMode": "numeric",
@@ -276,7 +276,7 @@
       "helpText": "First CPU Idle marks the first time at which the page's main thread is quiet enough to handle input. [Learn more](https://developers.google.com/web/tools/lighthouse/audits/first-interactive)."
     },
     "consistently-interactive": {
-      "score": 0.82,
+      "score": 0.69,
       "displayValue": "4,930 ms",
       "rawValue": 4927.278,
       "extendedInfo": {
@@ -2470,7 +2470,7 @@
       "helpText": "DOM order matches the visual order, improving navigation for assistive technology. [Learn more](https://developers.google.com/web/fundamentals/accessibility/how-to-review#try_it_with_a_screen_reader)."
     },
     "uses-long-cache-ttl": {
-      "score": 0.91,
+      "score": 0.58,
       "displayValue": "11 assets found",
       "rawValue": 103455,
       "extendedInfo": {
@@ -2570,7 +2570,7 @@
       },
       "scoreDisplayMode": "numeric",
       "name": "uses-long-cache-ttl",
-      "description": "Uses efficient cache policy on static assets",
+      "description": "Uses inefficient cache policy on static assets",
       "helpText": "A long cache lifetime can speed up repeat visits to your page. [Learn more](https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/http-caching#cache-control).",
       "details": {
         "type": "table",
@@ -4754,7 +4754,7 @@
         }
       ],
       "id": "performance",
-      "score": 0.7
+      "score": 0.62
     },
     {
       "name": "Progressive Web App",