PWA Audits: add placeholders for rest of baseline checks. (#2248)

* Add manual PWA placeholder audits

* Add tests, update tests, fix tests.

* Add additional test

* feedback

lighthouse-core/audits/audit.js

@@ -147,6 +147,7 @@ class Audit {
       extendedInfo: result.extendedInfo,
       scoringMode: audit.meta.scoringMode || Audit.SCORING_MODES.BINARY,
       informative: audit.meta.informative,
+      manual: audit.meta.manual,
       name: audit.meta.name,
       category: audit.meta.category,
       description: audit.meta.description,

lighthouse-core/audits/manual/manual-audit.js

@@ -0,0 +1,51 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+/**
+ * @fileoverview Base class for audits that the user should verify manually on
+ * their site.
+ */
+
+const Audit = require('../audit');
+
+class ManualAudit extends Audit {
+
+  /**
+   * @return {!AuditMeta}
+   */
+  static get meta() {
+    return {
+      informative: true,
+      manual: true,
+      requiredArtifacts: []
+    };
+  }
+
+  /**
+   * @return {!AuditResult}
+   */
+  static audit() {
+    return {
+      rawValue: false,
+      // displayValue: '(needs manual verification)'
+    };
+  }
+}
+
+module.exports = ManualAudit;

lighthouse-core/audits/manual/pwa-cross-browser.js

@@ -0,0 +1,43 @@
+
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const ManualAudit = require('./manual-audit');
+
+/**
+ * @fileoverview Manual PWA audit for cross browser support.
+ */
+
+class PWACrossBrowser extends ManualAudit {
+
+  /**
+   * @return {!AuditMeta}
+   */
+  static get meta() {
+    return Object.assign({
+      category: 'PWA',
+      name: 'pwa-cross-browser',
+      helpText: 'To reach the most number of users, sites should work across ' +
+      'every major browser. [Learn more](https://developers.google.com/web/progressive-web-apps/checklist#site-works-cross-browser).',
+      description: 'Site works cross-browser',
+    }, super.meta);
+  }
+}
+
+module.exports = PWACrossBrowser;

lighthouse-core/audits/manual/pwa-each-page-has-url.js

@@ -0,0 +1,42 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const ManualAudit = require('./manual-audit');
+
+/**
+ * @fileoverview Manual PWA audit to ensure every page has a deep link.
+ */
+
+class PWAEachPageHasURL extends ManualAudit {
+
+  /**
+   * @return {!AuditMeta}
+   */
+  static get meta() {
+    return Object.assign({
+      category: 'PWA',
+      name: 'pwa-each-page-has-url',
+      helpText: 'Ensure individual pages are deep linkable via the URLs and that URLs are ' +
+          'unique for the purpose of shareability on social media. [Learn more](https://developers.google.com/web/progressive-web-apps/checklist#each-page-has-a-url).',
+      description: 'Each page has a URL',
+    }, super.meta);
+  }
+}
+
+module.exports = PWAEachPageHasURL;

lighthouse-core/audits/manual/pwa-page-transitions.js

@@ -0,0 +1,42 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const ManualAudit = require('./manual-audit');
+
+/**
+ * @fileoverview Manual PWA audit for janky-free page transitions.
+ */
+
+class PWAPageTransitions extends ManualAudit {
+
+  /**
+   * @return {!AuditMeta}
+   */
+  static get meta() {
+    return Object.assign({
+      category: 'PWA',
+      name: 'pwa-page-transitions',
+      helpText: 'Transitions should feel snappy as you tap around, even on a slow network, a key ' +
+          'to perceived performance. [Learn more](https://developers.google.com/web/progressive-web-apps/checklist#page-transitions-dont-feel-like-they-block-on-the-network).',
+      description: 'Page transitions don\'t feel like they block on the network',
+    }, super.meta);
+  }
+}
+
+module.exports = PWAPageTransitions;

lighthouse-core/config/default.js

@@ -76,6 +76,9 @@ module.exports = {
     "manifest-short-name-length",
     "content-width",
     "deprecations",
+    "manual/pwa-cross-browser",
+    "manual/pwa-page-transitions",
+    "manual/pwa-each-page-has-url",
     "accessibility/accesskeys",
     "accessibility/aria-allowed-attr",
     "accessibility/aria-required-attr",
@@ -136,7 +139,7 @@ module.exports = {
   "aggregations": [{
     "name": "Progressive Web App",
     "id": "pwa",
-    "description": "These audits validate the aspects of a Progressive Web App. They are a subset of the [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist).",
+    "description": "These audits validate the aspects of a Progressive Web App. They are a subset of the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist).",
     "scored": true,
     "categorizable": true,
     "items": [{
@@ -606,42 +609,48 @@ module.exports = {
     },
     "a11y-color-contrast": {
       "title": "Color Contrast Is Satisfactory",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
     },
     "a11y-describe-contents": {
       "title": "Elements Describe Contents Well",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
     },
     "a11y-well-structured": {
       "title": "Elements Are Well Structured",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
     },
     "a11y-aria": {
       "title": "ARIA Attributes Follow Best Practices",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
     },
     "a11y-correct-attributes": {
       "title": "Elements Use Attributes Correctly",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
     },
     "a11y-element-names": {
       "title": "Elements Have Discernable Names",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
     },
     "a11y-language": {
       "title": "Page Specifies Valid Language",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
     },
     "a11y-meta": {
       "title": "Meta Tags Used Properly",
-      "description": "Screen readers and other assitive technologies require annotations to understand otherwise ambiguous content."
+      "description": "Screen readers and other assistive technologies require annotations to understand otherwise ambiguous content."
+    },
+    "manual-pwa-checks": {
+      "title": "Manual checks to verify",
+      "description": "These audits are required by the baseline " +
+          "[PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist) but are " +
+          "not automatically checked by Lighthouse. They do not affect your score but it's important that you verify them manually."
     },
   },
   "categories": {
     "pwa": {
       "name": "Progressive Web App",
       "weight": 1,
-      "description": "These audits validate the aspects of a Progressive Web App. They are a subset of the [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist).",
+      "description": "These audits validate the aspects of a Progressive Web App, as specified by the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist).",
       "audits": [
         {"id": "service-worker", "weight": 1},
         {"id": "works-offline", "weight": 1},
@@ -653,7 +662,10 @@ module.exports = {
         {"id": "splash-screen", "weight": 1},
         {"id": "themed-omnibox", "weight": 1},
         {"id": "viewport", "weight": 1},
-        {"id": "content-width", "weight": 1}
+        {"id": "content-width", "weight": 1},
+        {"id": "pwa-cross-browser", "weight": 0, "group": "manual-pwa-checks"},
+        {"id": "pwa-page-transitions", "weight": 0, "group": "manual-pwa-checks"},
+        {"id": "pwa-each-page-has-url", "weight": 0, "group": "manual-pwa-checks"}
       ]
     },
     "performance": {

lighthouse-core/report/v2/renderer/category-renderer.js

@@ -66,6 +66,9 @@ class CategoryRenderer {
     if (audit.result.informative) {
       scoreEl.classList.add('lh-score--informative');
     }
+    if (audit.result.manual) {
+      scoreEl.classList.add('lh-score--manual');
+    }
 
     return this._populateScore(tmpl, audit.score, scoringMode, title, description);
   }
@@ -185,7 +188,7 @@ class CategoryRenderer {
     auditGroupHeader.textContent = group.title;
 
     const auditGroupDescription = this._dom.createElement('div', 'lh-audit-group__description');
-    auditGroupDescription.textContent = group.description;
+    auditGroupDescription.appendChild(this._dom.createSpanFromMarkdown(group.description));
 
     const auditGroupSummary = this._dom.createElement('summary',
           'lh-audit-group__summary lh-expandable-details__summary');
@@ -213,6 +216,35 @@ class CategoryRenderer {
     return passedElem;
   }
 
+  /**
+   * @param {!Array<!ReportRenderer.AuditJSON>} manualAudits
+   * @param {!Object<string, !ReportRenderer.GroupJSON>} groupDefinitions
+   * @param {!Element} element Parent container to add the manual audits to.
+   */
+  _renderManualAudits(manualAudits, groupDefinitions, element) {
+    const auditsGroupedByGroup = /** @type {!Object<string,
+        !Array<!ReportRenderer.AuditJSON>>} */ ({});
+    manualAudits.forEach(audit => {
+      const group = auditsGroupedByGroup[audit.group] || [];
+      group.push(audit);
+      auditsGroupedByGroup[audit.group] = group;
+    });
+
+    Object.keys(auditsGroupedByGroup).forEach(groupId => {
+      const group = groupDefinitions[groupId];
+      const auditGroupElem = this._renderAuditGroup(group);
+
+      this._dom.find('.lh-audit-group__summary', auditGroupElem)
+          .classList.add('lh-audit-group__summary--manual');
+
+      auditsGroupedByGroup[groupId].forEach(audit => {
+        auditGroupElem.appendChild(this._renderAudit(audit));
+      });
+
+      element.appendChild(auditGroupElem);
+    });
+  }
+
   /**
    * @param {!Document|!Element} context
    */
@@ -262,34 +294,39 @@ class CategoryRenderer {
       case 'accessibility':
         return this._renderAccessibilityCategory(category, groups);
       default:
-        return this._renderDefaultCategory(category);
+        return this._renderDefaultCategory(category, groups);
     }
   }
 
   /**
    * @param {!ReportRenderer.CategoryJSON} category
+   * @param {!Object<string, !ReportRenderer.GroupJSON>} groupDefinitions
    * @return {!Element}
    */
-  _renderDefaultCategory(category) {
+  _renderDefaultCategory(category, groupDefinitions) {
     const element = this._dom.createElement('div', 'lh-category');
     element.id = category.id;
     element.appendChild(this._renderCategoryScore(category));
 
-    const passedAudits = category.audits.filter(audit => audit.score === 100);
-    const nonPassedAudits = category.audits.filter(audit => !passedAudits.includes(audit));
+    const manualAudits = category.audits.filter(audit => audit.result.manual);
+    const nonManualAudits = category.audits.filter(audit => !manualAudits.includes(audit));
+    const passedAudits = nonManualAudits.filter(audit => audit.score === 100);
+    const nonPassedAudits = nonManualAudits.filter(audit => !passedAudits.includes(audit));
 
     for (const audit of nonPassedAudits) {
       element.appendChild(this._renderAudit(audit));
     }
 
-    // Don't create a passed section if there are no passed.
-    if (!passedAudits.length) {
-      return element;
+    // Create a passed section if there are passing audits.
+    if (passedAudits.length) {
+      const passedElements = passedAudits.map(audit => this._renderAudit(audit));
+      const passedElem = this._renderPassedAuditsSection(passedElements);
+      element.appendChild(passedElem);
     }
 
-    const passedElements = passedAudits.map(audit => this._renderAudit(audit));
-    const passedElem = this._renderPassedAuditsSection(passedElements);
-    element.appendChild(passedElem);
+    // Render manual audits after passing.
+    this._renderManualAudits(manualAudits, groupDefinitions, element);
+
     return element;
   }
 

lighthouse-core/report/v2/renderer/report-renderer.js

@@ -176,6 +176,7 @@ if (typeof module !== 'undefined' && module.exports) {
  *       rawValue: (number|undefined),
  *       description: string,
  *       informative: boolean,
+ *       manual: boolean,
  *       debugString: string,
  *       displayValue: string,
  *       helpText: string,