core(installable-manifest): always run InstallabilityErrors in legacy mode

.github/workflows/smoke.yml

@@ -123,7 +123,7 @@ jobs:
 
     - run: sudo apt-get install xvfb
     - name: yarn smoke --fraggle-rock
-      run: xvfb-run --auto-servernum yarn smoke --debug --fraggle-rock -j=1 --retries=2 --invert-match pwa offline
+      run: xvfb-run --auto-servernum yarn smoke --debug --fraggle-rock -j=1 --retries=2
 
     # Fail if any changes were written to source files.
     - run: git diff --exit-code

lighthouse-cli/test/smokehouse/test-definitions/offline-online-only.js

@@ -68,7 +68,7 @@ const expectations = {
       },
       'installable-manifest': {
         score: 0,
-        details: {items: [{reason: 'No manifest was fetched'}]},
+        details: {items: [{reason: 'Page has no manifest <link> URL'}]},
       },
       'splash-screen': {
         score: 0,

lighthouse-cli/test/smokehouse/test-definitions/pwa-chromestatus.js

@@ -30,7 +30,7 @@ const expectations = {
       },
       'installable-manifest': {
         score: 0,
-        details: {items: [{reason: 'No manifest was fetched'}]},
+        details: {items: [{reason: 'Page has no manifest <link> URL'}]},
       },
       'splash-screen': {
         score: 0,

lighthouse-core/audits/installable-manifest.js

@@ -125,7 +125,7 @@ class InstallableManifest extends Audit {
       failureTitle: str_(UIStrings.failureTitle),
       description: str_(UIStrings.description),
       supportedModes: ['navigation'],
-      requiredArtifacts: ['URL', 'WebAppManifest', 'InstallabilityErrors'],
+      requiredArtifacts: ['WebAppManifest', 'InstallabilityErrors'],
     };
   }
 
@@ -195,7 +195,6 @@ class InstallableManifest extends Audit {
    *
    */
   static async audit(artifacts, context) {
-    const manifestValues = await ManifestValues.request(artifacts, context);
     const {i18nErrors, warnings} = InstallableManifest.getInstallabilityErrors(artifacts);
 
     const manifestUrl = artifacts.WebAppManifest ? artifacts.WebAppManifest.url : null;
@@ -210,10 +209,16 @@ class InstallableManifest extends Audit {
     const errorReasons = i18nErrors.map(reason => {
       return {reason};
     });
-    /** DevTools InstallabilityErrors does not emit an error unless there is a manifest, so include manifestValues's error */
-    if (manifestValues.isParseFailure) {
-      errorReasons.push({
-        reason: manifestValues.parseFailureReason});
+
+    // If InstallabilityErrors is empty, double check ManifestValues to make sure nothing was missed.
+    // InstallabilityErrors can be empty erroneously in our DevTools web tests.
+    if (!errorReasons.length) {
+      const manifestValues = await ManifestValues.request(artifacts, context);
+      if (manifestValues.isParseFailure) {
+        errorReasons.push({
+          reason: manifestValues.parseFailureReason,
+        });
+      }
     }
 
     // Include the detailed pass/fail checklist as a diagnostic.

lighthouse-core/gather/gather-runner.js

@@ -438,10 +438,8 @@ class GatherRunner {
     }
 
     try {
-      if (baseArtifacts.WebAppManifest) {
-        baseArtifacts.InstallabilityErrors = await InstallabilityErrors.getInstallabilityErrors(
-          passContext.driver.defaultSession);
-      }
+      baseArtifacts.InstallabilityErrors = await InstallabilityErrors.getInstallabilityErrors(
+        passContext.driver.defaultSession);
     } catch (err) {
       log.error('GatherRunner InstallabilityErrors', err);
       baseArtifacts.InstallabilityErrors = {

lighthouse-core/test/audits/installable-manifest-test.js

@@ -59,11 +59,12 @@ describe('PWA: webapp install banner audit', () => {
 
     it('fails with a non-parsable manifest', () => {
       const artifacts = generateMockArtifacts('{,:}');
+      artifacts.InstallabilityErrors.errors.push({errorId: 'manifest-empty', errorArguments: []});
       const context = generateMockAuditContext();
       return InstallableManifestAudit.audit(artifacts, context).then(result => {
         assert.strictEqual(result.score, 0);
         const items = result.details.items;
-        assert.ok(items[0].reason.includes('failed to parse as valid JSON'));
+        expect(items[0].reason).toBeDisplayString(/could not be parsed/);
       });
     });
 

lighthouse-core/test/results/artifacts/artifacts.json

@@ -7,7 +7,12 @@
   "BenchmarkIndex": 1549,
   "WebAppManifest": null,
   "InstallabilityErrors": {
-    "errors": []
+    "errors": [
+      {
+        "errorId": "no-manifest",
+        "errorArguments": []
+      }
+    ]
   },
   "Stacks": [
     {

lighthouse-core/test/results/sample_v2.json

@@ -799,7 +799,7 @@
         ],
         "items": [
           {
-            "reason": "No manifest was fetched"
+            "reason": "Page has no manifest <link> URL"
           }
         ],
         "debugData": {
@@ -7143,19 +7143,19 @@
       },
       {
         "startTime": 0,
-        "name": "lh:computed:ManifestValues",
+        "name": "lh:audit:apple-touch-icon",
         "duration": 100,
         "entryType": "measure"
       },
       {
         "startTime": 0,
-        "name": "lh:audit:apple-touch-icon",
+        "name": "lh:audit:splash-screen",
         "duration": 100,
         "entryType": "measure"
       },
       {
         "startTime": 0,
-        "name": "lh:audit:splash-screen",
+        "name": "lh:computed:ManifestValues",
         "duration": 100,
         "entryType": "measure"
       },
@@ -8388,6 +8388,9 @@
       "lighthouse-core/audits/installable-manifest.js | columnValue": [
         "audits[installable-manifest].details.headings[0].text"
       ],
+      "lighthouse-core/audits/installable-manifest.js | no-manifest": [
+        "audits[installable-manifest].details.items[0].reason"
+      ],
       "lighthouse-core/audits/apple-touch-icon.js | failureTitle": [
         "audits[apple-touch-icon].title"
       ],

third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run-expected.txt

@@ -127,6 +127,7 @@ Gathering: SourceMaps
 Gathering: FullPageScreenshot
 Populate base artifacts
 Get webapp manifest
+Get webapp installability errors
 Collect stacks
 Running offlinePass pass
 Resetting state with about:blank