From 45b70e0ed53c7013706eb06999c1485f25032083 Mon Sep 17 00:00:00 2001
From: Brendan Kenny <bckenny@google.com>
Date: Mon, 5 Jun 2017 16:34:52 -0700
Subject: [PATCH] update smokehouse PWA expectations

---
 lighthouse-cli/test/smokehouse/pwa-config.js  |  42 +-
 .../test/smokehouse/pwa-expectations.js       | 447 ++++++++++++++++--
 .../gather/computed/manifest-values.js        |   2 +-
 3 files changed, 400 insertions(+), 91 deletions(-)

diff --git a/lighthouse-cli/test/smokehouse/pwa-config.js b/lighthouse-cli/test/smokehouse/pwa-config.js
index 45db5e995371..8144b17b5dd5 100644
--- a/lighthouse-cli/test/smokehouse/pwa-config.js
+++ b/lighthouse-cli/test/smokehouse/pwa-config.js
@@ -20,42 +20,8 @@
  * Config file for running PWA smokehouse audits.
  */
 module.exports = {
-  passes: [{
-    recordTrace: true,
-    gatherers: [
-      'url',
-      'theme-color',
-      'manifest',
-      // https://github.com/GoogleChrome/lighthouse/issues/566
-      // 'cache-contents'
-    ]
-  },
-  {
-    passName: 'offlinePass',
-    gatherers: [
-      'service-worker',
-      'offline',
-      'start-url'
-    ]
-  },
-  {
-    passName: 'domstats',
-    gatherers: [
-      'dobetterweb/domstats',
-      'http-redirect'
-    ]
-  }],
-
-  audits: [
-    'dobetterweb/dom-size',
-    'is-on-https',
-    'redirects-http',
-    'service-worker',
-    'works-offline',
-    'webapp-install-banner',
-    'splash-screen',
-    'themed-omnibox',
-    // https://github.com/GoogleChrome/lighthouse/issues/566
-    // 'cache-start-url'
-  ]
+  extends: 'lighthouse:default',
+  settings: {
+    onlyCategories: ['pwa']
+  }
 };
diff --git a/lighthouse-cli/test/smokehouse/pwa-expectations.js b/lighthouse-cli/test/smokehouse/pwa-expectations.js
index e26bfb204b7c..7c5b93018acd 100644
--- a/lighthouse-cli/test/smokehouse/pwa-expectations.js
+++ b/lighthouse-cli/test/smokehouse/pwa-expectations.js
@@ -37,18 +37,92 @@ module.exports = [
       'works-offline': {
         score: true
       },
-      'webapp-install-banner': {
+      'viewport': {
         score: true
       },
-      'splash-screen': {
+      'without-javascript': {
         score: true
       },
+      'load-fast-enough-for-pwa': {
+        // Ignore speed test; just verify that it ran.
+      },
+      'webapp-install-banner': {
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: true},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
+      },
+      'splash-screen': {
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: true},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
+      },
       'themed-omnibox': {
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: true},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
+      },
+      'content-width': {
         score: true
       },
-      // 'cache-start-url': {
-      //   score: true
-      // }
+
+      // "manual" audits. Just verify in the results.
+      'pwa-cross-browser': {
+        score: false,
+        manual: true
+      },
+      'pwa-page-transitions': {
+        score: false,
+        manual: true
+      },
+      'pwa-each-page-has-url': {
+        score: false,
+        manual: true
+      }
     }
   },
 
@@ -56,15 +130,6 @@ module.exports = [
     initialUrl: 'https://www.chromestatus.com/',
     url: 'https://www.chromestatus.com/features',
     audits: {
-      'dom-size': {
-        score: 100,
-        extendedInfo: {
-          value: {
-            1: {value: '20'},
-            2: {snippet: /ul.versionlist/}
-          }
-        }
-      },
       'is-on-https': {
         score: true
       },
@@ -77,18 +142,92 @@ module.exports = [
       'works-offline': {
         score: false
       },
+      'viewport': {
+        score: true
+      },
+      'without-javascript': {
+        score: true
+      },
+      'load-fast-enough-for-pwa': {
+        // Ignore speed test; just verify that it ran.
+      },
       'webapp-install-banner': {
-        score: false
+        score: false,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: false},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
       },
       'splash-screen': {
-        score: true
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: false},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
       },
       'themed-omnibox': {
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: false},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
+      },
+      'content-width': {
         score: true
       },
-      // 'cache-start-url': {
-      //   score: true
-      // }
+
+      // "manual" audits. Just verify in the results.
+      'pwa-cross-browser': {
+        score: false,
+        manual: true
+      },
+      'pwa-page-transitions': {
+        score: false,
+        manual: true
+      },
+      'pwa-each-page-has-url': {
+        score: false,
+        manual: true
+      }
     }
   },
 
@@ -100,7 +239,9 @@ module.exports = [
         score: true
       },
       'redirects-http': {
-        score: true
+        // Note: relies on JS redirect.
+        // see https://github.com/GoogleChrome/lighthouse/issues/2383
+        score: false
       },
       'service-worker': {
         score: true
@@ -108,21 +249,97 @@ module.exports = [
       'works-offline': {
         score: true
       },
+      'viewport': {
+        score: true
+      },
+      'without-javascript': {
+        score: true
+      },
+      'load-fast-enough-for-pwa': {
+        // Ignore speed test; just verify that it ran.
+      },
       'webapp-install-banner': {
-        score: false
+        score: false,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: false},
+                {id: 'shortNameLength', passing: false},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
       },
       'splash-screen': {
-        score: true
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: false},
+                {id: 'shortNameLength', passing: false},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
       },
       'themed-omnibox': {
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: true},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: false},
+                {id: 'shortNameLength', passing: false},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
+      },
+      'content-width': {
         score: true
       },
-      // 'cache-start-url': {
-      //   score: true
-      // }
+
+      // "manual" audits. Just verify in the results.
+      'pwa-cross-browser': {
+        score: false,
+        manual: true
+      },
+      'pwa-page-transitions': {
+        score: false,
+        manual: true
+      },
+      'pwa-each-page-has-url': {
+        score: false,
+        manual: true
+      }
     }
   },
 
+  // Disabled due to flakiness of site.
+  // See https://github.com/GoogleChrome/lighthouse/issues/1656
   // {
   //   initialUrl: 'https://shop.polymer-project.org/',
   //   url: 'https://shop.polymer-project.org/',
@@ -139,40 +356,92 @@ module.exports = [
   //     'works-offline': {
   //       score: true
   //     },
-  //     'manifest-display': {
-  //       score: true,
-  //       displayValue: 'standalone'
+  //     'viewport': {
+  //       score: true
   //     },
-  //     'manifest-exists': {
+  //     'without-javascript': {
   //       score: true
   //     },
-  //     'manifest-background-color': {
+  //     'load-fast-enough-for-pwa': {
+  //       // Ignore speed test; just verify that it ran.
+  //     },
+  //     'webapp-install-banner': {
   //       score: true,
   //       extendedInfo: {
-  //         value: '#fff'
+  //         value: {
+  //           manifestValues: {
+  //             allChecks: [
+  //               {id: 'hasStartUrl', passing: true},
+  //               {id: 'hasIconsAtLeast192px', passing: true},
+  //               {id: 'hasIconsAtLeast512px', passing: true},
+  //               {id: 'hasPWADisplayValue', passing: true},
+  //               {id: 'hasBackgroundColor', passing: true},
+  //               {id: 'hasThemeColor', passing: true},
+  //               {id: 'hasShortName', passing: true},
+  //               {id: 'shortNameLength', passing: true},
+  //               {id: 'hasName', passing: true}
+  //             ]
+  //           }
+  //         }
   //       }
   //     },
-  //     'manifest-theme-color': {
-  //       score: true
-  //     },
-  //     'manifest-icons-min-192': {
-  //       score: true
+  //     'splash-screen': {
+  //       score: true,
+  //       extendedInfo: {
+  //         value: {
+  //           manifestValues: {
+  //             allChecks: [
+  //               {id: 'hasStartUrl', passing: true},
+  //               {id: 'hasIconsAtLeast192px', passing: true},
+  //               {id: 'hasIconsAtLeast512px', passing: true},
+  //               {id: 'hasPWADisplayValue', passing: true},
+  //               {id: 'hasBackgroundColor', passing: true},
+  //               {id: 'hasThemeColor', passing: true},
+  //               {id: 'hasShortName', passing: true},
+  //               {id: 'shortNameLength', passing: true},
+  //               {id: 'hasName', passing: true}
+  //             ]
+  //           }
+  //         }
+  //       }
   //     },
-  //     'manifest-icons-min-144': {
-  //       score: true
+  //     'themed-omnibox': {
+  //       score: true,
+  //       extendedInfo: {
+  //         value: {
+  //           manifestValues: {
+  //             allChecks: [
+  //               {id: 'hasStartUrl', passing: true},
+  //               {id: 'hasIconsAtLeast192px', passing: true},
+  //               {id: 'hasIconsAtLeast512px', passing: true},
+  //               {id: 'hasPWADisplayValue', passing: true},
+  //               {id: 'hasBackgroundColor', passing: true},
+  //               {id: 'hasThemeColor', passing: true},
+  //               {id: 'hasShortName', passing: true},
+  //               {id: 'shortNameLength', passing: true},
+  //               {id: 'hasName', passing: true}
+  //             ]
+  //           }
+  //         }
+  //       }
   //     },
-  //     'manifest-name': {
+  //     'content-width': {
   //       score: true
   //     },
-  //     'manifest-short-name': {
-  //       score: true
+
+  //     // "manual" audits. Just verify in the results.
+  //     'pwa-cross-browser': {
+  //       score: false,
+  //       manual: true
   //     },
-  //     'manifest-start-url': {
-  //       score: true
+  //     'pwa-page-transitions': {
+  //       score: false,
+  //       manual: true
   //     },
-  //     // 'cache-start-url': {
-  //     //   score: true
-  //     // }
+  //     'pwa-each-page-has-url': {
+  //       score: false,
+  //       manual: true
+  //     }
   //   }
   // },
 
@@ -192,18 +461,92 @@ module.exports = [
       'works-offline': {
         score: true
       },
-      'webapp-install-banner': {
+      'viewport': {
+        score: true
+      },
+      'without-javascript': {
         score: true
       },
+      'load-fast-enough-for-pwa': {
+        // Ignore speed test; just verify that it ran .
+      },
+      'webapp-install-banner': {
+        score: true,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: false},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: true},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
+      },
       'splash-screen': {
-        score: false
+        score: false,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: false},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: true},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
       },
       'themed-omnibox': {
-        score: false
+        score: false,
+        extendedInfo: {
+          value: {
+            manifestValues: {
+              allChecks: [
+                {id: 'hasStartUrl', passing: true},
+                {id: 'hasIconsAtLeast192px', passing: true},
+                {id: 'hasIconsAtLeast512px', passing: false},
+                {id: 'hasPWADisplayValue', passing: true},
+                {id: 'hasBackgroundColor', passing: true},
+                {id: 'hasThemeColor', passing: true},
+                {id: 'hasShortName', passing: true},
+                {id: 'shortNameLength', passing: true},
+                {id: 'hasName', passing: true}
+              ]
+            }
+          }
+        }
+      },
+      'content-width': {
+        score: true
+      },
+
+      // "manual" audits. Just verify in the results.
+      'pwa-cross-browser': {
+        score: false,
+        manual: true
+      },
+      'pwa-page-transitions': {
+        score: false,
+        manual: true
       },
-      // 'cache-start-url': {
-      //   score: true
-      // }
+      'pwa-each-page-has-url': {
+        score: false,
+        manual: true
+      }
     }
   }
 ];
diff --git a/lighthouse-core/gather/computed/manifest-values.js b/lighthouse-core/gather/computed/manifest-values.js
index 340665f66b3f..5c226ba2118a 100644
--- a/lighthouse-core/gather/computed/manifest-values.js
+++ b/lighthouse-core/gather/computed/manifest-values.js
@@ -67,7 +67,7 @@ class ManifestValues extends ComputedArtifact {
       {
         id: 'shortNameLength',
         failureText: 'Manifest `short_name` will be truncated when displayed on the homescreen',
-        validate: manifest => manifest.value.short_name.value &&
+        validate: manifest => !!manifest.value.short_name.value &&
             manifest.value.short_name.value.length <= SUGGESTED_SHORTNAME_LENGTH
       },
       {