Add TTI < 10s audit for PWA (#1840)

lighthouse-core/audits/load-fast-enough-for-pwa.js

@@ -0,0 +1,101 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+/** @fileoverview
+ *  This audit evaluates if a page's load performance is fast enough for it to be considered a PWA.
+ *  We are doublechecking that the network requests were throttled (or slow on their own)
+ *  Afterwards, we report if the TTI is less than 10 seconds.
+ */
+
+const Audit = require('./audit');
+const TTIMetric = require('./time-to-interactive');
+const Emulation = require('../lib/emulation');
+
+const Formatter = require('../report/formatter');
+
+// Maximum TTI to be considered "fast" for PWA baseline checklist
+//   https://developers.google.com/web/progressive-web-apps/checklist
+const MAXIMUM_TTI = 10 * 1000;
+
+class LoadFastEnough4Pwa extends Audit {
+  /**
+   * @return {!AuditMeta}
+   */
+  static get meta() {
+    return {
+      category: 'PWA',
+      name: 'load-fast-enough-for-pwa',
+      description: 'Page load is fast enough on 3G',
+      helpText: 'Satisfied if the _Time To Interactive_ duration is shorter than _10 seconds_, as defined by the [PWA Baseline Checklist](https://developers.google.com/web/progressive-web-apps/checklist). Network throttling is required (specifically: RTT latencies >= 150 RTT are expected).',
+      requiredArtifacts: ['traces', 'networkRecords']
+    };
+  }
+
+  /**
+   * @param {!Artifacts} artifacts
+   * @return {!AuditResult}
+   */
+  static audit(artifacts) {
+    const networkRecords = artifacts.networkRecords[Audit.DEFAULT_PASS];
+    const allRequestLatencies = networkRecords.map(record => {
+      if (!record._timing) return undefined;
+      // Use DevTools' definition of Waiting latency: https://github.com/ChromeDevTools/devtools-frontend/blob/66595b8a73a9c873ea7714205b828866630e9e82/front_end/network/RequestTimingView.js#L164
+      return record._timing.receiveHeadersEnd - record._timing.sendEnd;
+    });
+
+    const latency3gMin = Emulation.settings.TYPICAL_MOBILE_THROTTLING_METRICS.latency - 10;
+    const areLatenciesAll3G = allRequestLatencies.every(val =>
+        val === undefined || val > latency3gMin);
+
+    return TTIMetric.audit(artifacts).then(ttiResult => {
+      const timeToInteractive = ttiResult.extendedInfo.value.timings.timeToInteractive;
+      const isFast = timeToInteractive < MAXIMUM_TTI;
+
+      const extendedInfo = {
+        formatter: Formatter.SUPPORTED_FORMATS.NULL,
+        value: {areLatenciesAll3G, allRequestLatencies, isFast, timeToInteractive}
+      };
+
+      if (!areLatenciesAll3G) {
+        return {
+          rawValue: false,
+          // eslint-disable-next-line max-len
+          debugString: `The Time To Interactive was found at ${ttiResult.displayValue}, however, the network request latencies were not sufficiently realistic, so the performance measurements cannot be trusted.`,
+          extendedInfo
+        };
+      }
+
+      if (!isFast) {
+        return {
+          rawValue: false,
+           // eslint-disable-next-line max-len
+          debugString: `Under 3G conditions, the Time To Interactive was at ${ttiResult.displayValue}. More details in the "Performance" section.`,
+          extendedInfo
+        };
+      }
+
+      return {
+        rawValue: true,
+        extendedInfo
+      };
+    });
+  }
+}
+
+module.exports = LoadFastEnough4Pwa;

lighthouse-core/config/default.json

@@ -64,6 +64,7 @@
     "manifest-display",
     "without-javascript",
     "first-meaningful-paint",
+    "load-fast-enough-for-pwa",
     "speed-index-metric",
     "estimated-input-latency",
     "time-to-interactive",
@@ -161,6 +162,10 @@
       "name": "Page load performance is fast",
       "description": "Users notice if sites and apps don't perform well. These top-level metrics capture the most important perceived performance concerns.",
       "audits": {
+        "load-fast-enough-for-pwa": {
+          "expectedValue": true,
+          "weight": 1
+        },
         "first-meaningful-paint": {
           "expectedValue": 100,
           "weight": 1

lighthouse-core/lib/emulation.js

@@ -151,5 +151,14 @@ module.exports = {
   enableCPUThrottling,
   disableCPUThrottling,
   goOffline,
-  getEmulationDesc
+  getEmulationDesc,
+  settings: {
+    NEXUS5X_EMULATION_METRICS,
+    NEXUS5X_USERAGENT,
+    TYPICAL_MOBILE_THROTTLING_METRICS,
+    OFFLINE_METRICS,
+    NO_THROTTLING_METRICS,
+    NO_CPU_THROTTLE_METRICS,
+    CPU_THROTTLE_METRICS
+  }
 };

lighthouse-core/test/audits/load-fast-enough-for-pwa-test.js

@@ -0,0 +1,102 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const FastPWAAudit = require('../../audits/load-fast-enough-for-pwa');
+const TTIAudit = require('../../audits/time-to-interactive');
+const Audit = require('../../audits/audit.js');
+const assert = require('assert');
+
+function generateTTIResults(ttiValue) {
+  const ttiResult = {
+    rawValue: ttiValue,
+    extendedInfo: {
+      value: {
+        timings: {
+          timeToInteractive: ttiValue
+        }
+      }
+    }
+  };
+  return Promise.resolve.bind(Promise, ttiResult);
+}
+
+function generateArtifacts(networkRecords = []) {
+  return {
+    networkRecords: {
+      [Audit.DEFAULT_PASS]: networkRecords
+    },
+    traces: {
+      [Audit.DEFAULT_PASS]: {traceEvents: []}
+    }
+  };
+}
+
+/* eslint-env mocha */
+describe('PWA: load-fast-enough-for-pwa audit', () => {
+  // monkeypatch TTI to for a more focused test
+  let origTTI;
+  beforeEach(() => origTTI = TTIAudit.audit);
+  afterEach(() => TTIAudit.audit = origTTI);
+
+  it('returns boolean based on TTI value', () => {
+    TTIAudit.audit = generateTTIResults(5000);
+    return FastPWAAudit.audit(generateArtifacts()).then(result => {
+      assert.equal(result.rawValue, true, 'fixture trace is not passing audit');
+    });
+  });
+
+  it('fails a bad TTI value', () => {
+    TTIAudit.audit = generateTTIResults(15000);
+    return FastPWAAudit.audit(generateArtifacts()).then(result => {
+      assert.equal(result.rawValue, false, 'not failing a long TTI value');
+      assert.ok(result.debugString);
+    });
+  });
+
+  it('fails a good TTI value with no throttling', () => {
+    TTIAudit.audit = generateTTIResults(5000);
+    // latencies are very short
+    const mockNetworkRecords = [
+      {_timing: {sendEnd: 0, receiveHeadersEnd: 50}},
+      {_timing: {sendEnd: 0, receiveHeadersEnd: 75}},
+      { },
+      {_timing: {sendEnd: 0, receiveHeadersEnd: 50}},
+    ];
+    return FastPWAAudit.audit(generateArtifacts(mockNetworkRecords)).then(result => {
+      assert.equal(result.rawValue, false);
+      assert.ok(result.debugString.includes('network request latencies'));
+    });
+  });
+
+
+  it('passes a good TTI value and WITH throttling', () => {
+    TTIAudit.audit = generateTTIResults(5000);
+    // latencies are very long
+    const mockNetworkRecords = [
+      {_timing: {sendEnd: 0, receiveHeadersEnd: 250}},
+      {_timing: {sendEnd: 0, receiveHeadersEnd: 175}},
+      { },
+      {_timing: {sendEnd: 0, receiveHeadersEnd: 250}},
+    ];
+    return FastPWAAudit.audit(generateArtifacts(mockNetworkRecords)).then(result => {
+      assert.equal(result.rawValue, true);
+      assert.strictEqual(result.debugString, undefined);
+    });
+  });
+});