Merge branch 'master' into cache_audit

.github/CODEOWNERS

@@ -0,0 +1,20 @@
+# Each line is a file pattern followed by one or more owners.
+# Order is important. The last matching pattern has the most precedence.
+# https://help.github.com/articles/about-codeowners/
+
+lighthouse-cli/                     @patrickhulce       @paulirish
+lighthouse-cli/test/fixtures/       @patrickhulce       @brendankenny
+lighthouse-cli/test/smokehouse/     @patrickhulce       @brendankenny
+
+lighthouse-core/gather/             @patrickhulce       @paulirish
+lighthouse-core/audits/             @patrickhulce       @brendankenny   @paulirish
+lighthouse-core/report/             @vinamratasingal    @paulirish
+
+lighthouse-core/config/             @patrickhulce       @brendankenny
+lighthouse-core/lib/                @brendankenny       @paulirish
+lighthouse-extension/               @paulirish          @patrickhulce
+lighthouse-viewer/                  @ebidel             @brendankenny
+
+docs/                               @vinamratasingal    @paulirish
+
+# no explicit owner for plots, travis, etc

CONTRIBUTING.md

@@ -6,6 +6,18 @@ We'd love your help! This doc covers how to become a contributor and submit code
 
 The `.eslintrc` file defines all. We use [JSDoc](http://usejsdoc.org/) along with [closure annotations](https://developers.google.com/closure/compiler/docs/js-for-compiler). Annotations are encouraged for all contributions.
 
+## Pull request titles
+
+We're using [conventional-commit](https://conventionalcommits.org/) for our commit messages. Since all PRs are squashed, we enforce this format for PR titles rather than individual git commits. A [`commitlint` bot](https://github.com/paulirish/commitlintbot) will update the status of your PR based on the title's conformance. The expected format is:
+
+> type(scope): message subject
+
+* The `type` must be one of: `new_audit` `core` `tests` `docs` `deps` `report` `cli` `extension` `misc`. (See [`.cz-config`](https://github.com/GoogleChrome/lighthouse/blob/master/.cz-config.js#L13))
+* The `scope` is optional, but recommended. Any string is allowed; it should indicate what the change affects.
+* The `message subject` should be pithy and direct.
+
+The [commitizen CLI](https://github.com/commitizen/cz-cli) can help to construct these commit messages.
+
 ## Learn about the architecture
 
 See [Lighthouse Architecture](./docs/architecture.md), our overview and tour of the codebase.
@@ -52,6 +64,42 @@ Don't:
 If no reference doc exists yet, then you can use the `helpText` as a stopgap for explaining
 both why the audit is important and how to fix it.
 
+## Tracking Errors
+
+We track our errors in the wild with Sentry. In general, do not worry about wrapping your audits or gatherers in try/catch blocks and reporting every error that could possibly occur; `lighthouse-core/runner.js` and `lighthouse-core/gather/gather-runner.js` already catch and report any errors that occur while running a gatherer or audit, including errors fatal to the entire run. However, there are some situations when you might want to expliticly handle an error and report it to Sentry or wrap it to avoid reporting. Generally, you can interact with Sentry simply by requiring the `lighthouse-core/lib/sentry.js` file and call its methods. The module exports a delegate that will correctly handle the error reporting based on the user's opt-in preference and will simply no-op if they haven't so you don't need to check.
+
+
+#### If you have an expected error that is recoverable but want to track how frequently it happens, *use Sentry.captureException*.
+
+```js
+const Sentry = require('./lighthouse-core/lib/sentry');
+
+try {
+  doRiskyThing();
+} catch (err) {
+  Sentry.captureException(err, {
+    tags: {audit: 'audit-name'},
+    level: 'warning',
+  });
+  doFallbackThing();
+}
+```
+
+#### If you need to track a code path that doesn't necessarily mean an error occurred, *use Sentry.captureMessage*.
+
+NOTE: If the message you're capturing is dynamic/based on user data or you need a stack trace, then create a fake error instead and use `Sentry.captureException` so that the instances will be grouped together in Sentry.
+
+```js
+const Sentry = require('./lighthouse-core/lib/sentry');
+
+if (networkRecords.length === 1) {
+  Sentry.captureMessage('Site only had 1 network request');
+  return null;
+} else {
+  // do your thang
+}
+```
+
 # For Maintainers
 
 ## Release guide

commitlint.config.js

@@ -0,0 +1,28 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+module.exports = {
+  extends: ['cz'],
+  rules: {
+    'body-leading-blank': [1, 'always'],
+    'body-tense': [1, 'always', ['present-imperative']],
+    'footer-leading-blank': [1, 'always'],
+    'footer-tense': [1, 'always', ['present-imperative']],
+    'header-max-length': [2, 'always', 80],
+    'lang': [0, 'always', 'eng'],
+    'scope-case': [2, 'always', 'lowerCase'],
+    'scope-empty': [0, 'never'],
+    'subject-case': [1, 'always', 'lowerCase'],
+    'subject-empty': [0, 'never'],
+    'subject-full-stop': [2, 'never', '.'],
+    'subject-tense': [1, 'always', ['present-imperative']],
+    'type-case': [2, 'always', 'lowerCase'],
+    'type-empty': [2, 'never'],
+    // The scope-enum :  defined in the cz-config
+    // The 'type-enum':  defined in the cz-config
+  },
+};

docs/bug-labels.md

@@ -30,6 +30,6 @@ Here are the different (actively used) labels and what they mean, organized by c
 - Question: question from community. Good fodder for new documentation that needs to be written. 
 
 ### Other labels
-- Good first bug: for new external contributor, these issues can be useful for them to tackle. 
+- Good first issue: for new external contributor, these issues can be useful for them to tackle. 
 - Help wanted: issues that could use help from the community. 
 

docs/error-reporting.md

@@ -0,0 +1,31 @@
+# Error Reporting Explained
+
+## What's going on?
+
+The Lighthouse team is constantly trying to improve the reliability of our tools, so we've added error tracking functionality to the CLI. Given your consent, we would like to anonymously report runtime exceptions using [Sentry](https://sentry.io/welcome/). We will use this information to detect new bugs and avoid regressions.
+
+Only CLI users are currently impacted. DevTools, extension, and node module users will not have errors reported.
+
+## What will happen if I opt-in?
+Runtime exceptions will be reported to the team along with information on your environment such as the URL you tested, your OS, and Chrome version. See [what data gets reported](#what-data-gets-reported).
+
+## What will happen if I do not opt-in?
+Runtime exceptions will not be reported to the team. Your ability to use Lighthouse will not be affected in any way.
+
+## What data gets reported?
+
+* The URL you tested
+* The runtime settings used (throttling enabled/disabled, emulation, etc)
+* The message, stack trace, and associated data of the error
+* The file path of Lighthouse node module on your machine
+* Your Lighthouse version
+* Your Chrome version
+* Your operating system
+
+## How do I opt-in?
+The first time you run the CLI you will be prompted with a message asking you if Lighthouse can anonymously report runtime exceptions. You can give a direct response of `yes` or `no` (`y`, `n`, and pressing enter which defaults to `no` are also acceptable responses), and you will not be prompted again. If no response is given within 20 seconds, a `no` response will be assumed, and you will not be prompted again. Non-interactive terminal sessions and invocations with the `CI` environment variable set will automatically not be prompted and will not opt-in by default.
+
+The CLI also has two flags to control error reporting that will override the saved preference. Running Lighthouse with `--enable-error-reporting` will report errors regardless of the saved preference, and running Lighthouse with `--no-enable-error-reporting` will *not* report errors regardless of the saved preferences.
+
+## How do I change my opt-in preference?
+Your response to the prompt will be saved to your home directory `~/.config/configstore/lighthouse.json` and used on future runs. To trigger a re-prompt, simply delete this file and Lighthouse will ask again on the next run. You can also edit this json file directly or run Lighthouse with the `--[no-]enable-error-reporting` flags.

docs/recipes/custom-audit/searchable-audit.js

@@ -17,7 +17,6 @@ const MAX_SEARCHABLE_TIME = 4000;
 class LoadAudit extends Audit {
   static get meta() {
     return {
-      category: 'MyCustomCategory',
       name: 'searchable-audit',
       description: 'Search box initialized and ready',
       failureDescription: 'Search box slow to initialize',

jsconfig.json

@@ -6,7 +6,9 @@
   "compilerOptions": {
     "target": "ES6",
     "noEmit": true,
-    "diagnostics": true
+    "diagnostics": true,
+    "checkJs": true,
+    "strict": true
   },
   "exclude": [
     "lighthouse-cli",

lighthouse-cli/bin.ts

@@ -5,6 +5,7 @@
  */
 'use strict';
 
+import {existsSync} from 'fs';
 import * as path from 'path';
 
 import * as Commands from './commands/commands';
@@ -15,10 +16,15 @@ import {runLighthouse} from './run';
 const log = require('lighthouse-logger');
 const perfOnlyConfig = require('../lighthouse-core/config/perf.json');
 const pkg = require('../package.json');
+const Sentry = require('../lighthouse-core/lib/sentry');
 
 // accept noop modules for these, so the real dependency is optional.
 import {updateNotifier} from './shim-modules';
+import {askPermission} from './sentry-prompt';
 
+function isDev() {
+  return existsSync(path.join(__dirname, '../.git'));
+}
 
 // Tell user if there's a newer version of LH.
 updateNotifier({pkg}).notify();
@@ -59,6 +65,23 @@ if (cliFlags.output === Printer.OutputMode[Printer.OutputMode.json] && !cliFlags
   cliFlags.outputPath = 'stdout';
 }
 
-export function run() {
+export async function run() {
+  if (typeof cliFlags.enableErrorReporting === 'undefined') {
+    cliFlags.enableErrorReporting = await askPermission();
+  }
+
+  Sentry.init({
+    url,
+    flags: cliFlags,
+    environmentData: {
+      name: 'redacted', // prevent sentry from using hostname
+      environment: isDev() ? 'development' : 'production',
+      release: pkg.version,
+      tags: {
+        channel: 'cli',
+      },
+    },
+  });
+
   return runLighthouse(url, cliFlags, config);
 }

lighthouse-cli/cli-flags.ts

@@ -14,7 +14,7 @@ import {GetValidOutputOptions, OutputMode} from './printer';
 export interface Flags {
   port: number, chromeFlags: string, output: any, outputPath: string, saveArtifacts: boolean,
       saveAssets: boolean, view: boolean, maxWaitForLoad: number, logLevel: string,
-      hostname: string, blockedUrlPatterns: string[]
+      hostname: string, blockedUrlPatterns: string[], enableErrorReporting: boolean
 }
 
 export function getFlags(manualArgv?: string) {
@@ -54,10 +54,12 @@ export function getFlags(manualArgv?: string) {
           [
             'save-assets', 'save-artifacts', 'list-all-audits', 'list-trace-categories',
             'additional-trace-categories', 'config-path', 'chrome-flags', 'perf', 'port',
-            'hostname', 'max-wait-for-load'
+            'hostname', 'max-wait-for-load', 'enable-error-reporting'
           ],
           'Configuration:')
       .describe({
+        'enable-error-reporting':
+            'Enables error reporting (prompts once by default, setting this flag will force error reporting to that state).',
         'blocked-url-patterns': 'Block any network requests to the specified URL patterns',
         'disable-storage-reset':
             'Disable clearing the browser cache and other storage APIs before a run',

lighthouse-cli/sentry-prompt.ts

@@ -0,0 +1,63 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+import {Configstore, inquirer} from './shim-modules';
+
+const log = require('lighthouse-logger');
+
+const MAXIMUM_WAIT_TIME = 20 * 1000;
+// clang-format off
+const MESSAGE =
+  `${log.reset}We're constantly trying to improve Lighthouse and its reliability.\n  ` +
+  `May we anonymously report runtime exceptions to improve the tool over time?\n  ` +
+  `${log.reset}Learn more: https://github.com/GoogleChrome/lighthouse/blob/master/docs/error-reporting.md`;
+// clang-format on
+
+async function prompt() {
+  if (!process.stdout.isTTY || process.env.CI) {
+    // Default non-interactive sessions to false
+    return false;
+  }
+
+  let timeout: NodeJS.Timer;
+
+  const prompt = inquirer.prompt([
+    {
+      type: 'confirm',
+      name: 'isErrorReportingEnabled',
+      default: false,
+      message: MESSAGE,
+    },
+  ]);
+
+  const timeoutPromise = new Promise((resolve: (a: boolean) => {}) => {
+    timeout = setTimeout(() => {
+      prompt.ui.close();
+      process.stdout.write('\n');
+      log.warn('CLI', 'No response to error logging preference, errors will not be reported.');
+      resolve(false);
+    }, MAXIMUM_WAIT_TIME);
+  });
+
+  return Promise.race([
+    prompt.then((result: {isErrorReportingEnabled: boolean}) => {
+      clearTimeout(timeout);
+      return result.isErrorReportingEnabled;
+    }),
+    timeoutPromise,
+  ]);
+}
+
+export async function askPermission() {
+  const configstore = new Configstore('lighthouse');
+  let isErrorReportingEnabled = configstore.get('isErrorReportingEnabled');
+  if (typeof isErrorReportingEnabled === 'boolean') {
+    return isErrorReportingEnabled;
+  }
+
+  isErrorReportingEnabled = await prompt();
+  configstore.set('isErrorReportingEnabled', isErrorReportingEnabled);
+  return isErrorReportingEnabled;
+}

lighthouse-cli/shim-modules.ts

@@ -30,4 +30,27 @@ try {
   };
 }
 
-export {opn, updateNotifier};
+let inquirer;
+try {
+  inquirer = require('inquirer');
+} catch (e) {
+  inquirer = {
+    prompt() {
+      console.error('module `inquirer` not installed. Not reporting errors.');
+      return Promise.resolve({isErrorReportingEnabled: false});
+    }
+  }
+}
+
+let Configstore;
+try {
+  Configstore = require('configstore');
+} catch (e) {
+  Configstore = function Configstore() {
+    console.error('module `configstore` not installed. Not persisting user choices.');
+    this.get = () => false;
+    this.set = () => undefined;
+  }
+}
+
+export {opn, updateNotifier, inquirer, Configstore};

lighthouse-cli/test/fixtures/accessibility/accessibility_tester.html → lighthouse-cli/test/fixtures/a11y/a11y_tester.html

@@ -120,7 +120,7 @@
     <section>
       <form>
         <input
-          id="label" 
+          id="label"
           type="text">
       </form>
     </section>
@@ -197,5 +197,13 @@
     <section>
       <video id="video-description"></video>
     </section>
+    <script>
+      // axe fails if a different UMD loader like almond.js is used on the page, see https://github.com/GoogleChrome/lighthouse/issues/2505
+      // We can simulate this behavior by defining a similarly misbehaving `define` function.
+      window.define = function () {
+        throw new Error('No AMD modules allowed');
+      };
+      window.define.amd = true;
+    </script>
   </body>
 </html>

lighthouse-cli/test/fixtures/dobetterweb/dbw_tester.html

@@ -35,6 +35,9 @@
 <!-- FAIL: block rendering -->
 <script src="./dbw_tester.js"></script>
 
+<!-- FAIL(errors-in-console): exception thrown -->
+<script type="text/javascript">throw new Error('An error');</script>
+
 <style>
   body {
     color: #000;
@@ -116,6 +119,12 @@ <h2>Do better web tester page</h2>
 
 </template>
 
+<!-- FAIL(image-aspect-ratio): image is naturally 480x318 -->
+<img src="lighthouse-480x318.jpg" width="480" height="57">
+<!-- PASS(image-aspect-ratio) -->
+<img src="lighthouse-480x318.jpg" width="480" height="318">
+
+
 <!-- Some websites overwrite the original Error object. The captureJSCallUsage function
   relies on the native Error object and prepareStackTrace from V8. When overwriting the stack
   property the e.stack inside the gatherer won't be in the correct format

lighthouse-cli/test/fixtures/seo/seo-failure-cases.html

@@ -13,6 +13,11 @@
   <!-- no <meta name="description" content=""> -->
 </head>
 <body>
-  bad crawlz
+  <h1>SEO</h1>
+
+  <h2>Anchor text</h2>
+  <a href='https://example.com'>click this</a>
+  <a href='/test.html'> click this </a>
+  <a href='/test.html'>CLICK THIS</a>
 </body>
 </html>