feat(cli): support Windows

.travis.yml

@@ -1,33 +1,51 @@
-sudo: required
-dist: xenial
-language: node_js
 branches:
   only:
     - master # Only build master and PRs
-matrix:
-  include:
-    - node_js: '10'
-    - node_js: '11'
 git:
   depth: false # ensure we have full git history for our build-context.js tests
 cache:
   yarn: true
   directories:
     - node_modules
-before_script:
-  - google-chrome-stable --version
-  - export CHROME_PATH="$(which google-chrome-stable)"
-  - export POSTGRES_DB_URL="postgres://postgres@localhost/lighthouse_ci_test"
-  - psql -c 'create database lighthouse_ci_test;' -U postgres
-script:
-  - yarn build
-  - yarn test
-  - yarn ci:dogfood
-before_cache:
-  # nyc, jest and other projects store files in here. They mess up the travis build cache.
-  - rm -rf ./node_modules/.cache/
-services:
-  - postgresql
-  - xvfb
-addons:
-  chrome: stable
+stages:
+  - test
+jobs:
+  include:
+    - &base-test
+      os: osx
+      language: node_js
+      node_js: v10
+      stage: test
+      script:
+        - yarn build
+        - yarn test:unit
+      before_cache:
+        # nyc, jest and other projects store files in here. They mess up the travis build cache.
+        - rm -rf ./node_modules/.cache/
+      addons:
+        chrome: stable
+    - <<: *base-test
+      os: windows
+      before_install:
+        - export LHCI_GITHUB_TOKEN="" # prevent any accidental leaking of the token
+        - choco install googlechrome
+      before_script:
+        - ps -W | sort
+      after_script:
+        - ps -W | sort
+      env: YARN_GPG=no
+    - <<: *base-test
+      os: linux
+      before_script:
+        - google-chrome-stable --version
+        - export CHROME_PATH="$(which google-chrome-stable)"
+        - export POSTGRES_DB_URL="postgres://postgres@localhost/lighthouse_ci_test"
+        - export RUN_E2E_TESTS="true"
+        - psql -c 'create database lighthouse_ci_test;' -U postgres
+      script:
+        - yarn build
+        - yarn test
+        - yarn ci:dogfood
+      services:
+        - postgresql
+        - xvfb

jest.config.js

@@ -6,6 +6,7 @@
 'use strict';
 
 module.exports = {
+  testRunner: require.resolve('jest-circus/runner'),
   transform: {
     '.*jsx$': 'babel-jest',
   },

package.json

@@ -55,6 +55,7 @@
     "eslint-plugin-react": "^7.13.0",
     "identity-obj-proxy": "^3.0.0",
     "jest": "^24.8.0",
+    "jest-circus": "^24.9.0",
     "jest-fetch-mock": "^2.1.2",
     "jest-image-snapshot": "^2.9.0",
     "lerna": "^3.16.4",

packages/cli/src/cli.js

@@ -117,7 +117,7 @@ async function run() {
 
 run().catch(err => {
   process.stderr.write(err.stack);
-  if (err.stdout) process.stderr.write(err.stdout);
-  if (err.stderr) process.stderr.write(err.stderr);
+  if (err.stdout) process.stderr.write('\n' + err.stdout.slice(0, 4000));
+  if (err.stderr) process.stderr.write('\n' + err.stderr);
   process.exit(1);
 });

packages/cli/src/collect/collect.js

@@ -75,9 +75,19 @@ async function determineUrls(options) {
   if (options.url) {
     let close = async () => undefined;
     if (options.startServerCommand) {
-      const {child} = await runCommandAndWaitForPattern(options.startServerCommand, /listen/);
+      const {child, patternMatch, stdout, stderr} = await runCommandAndWaitForPattern(
+        options.startServerCommand,
+        /(listen|ready)/i,
+        {timeout: 10000}
+      );
       process.stdout.write(`Started a web server with "${options.startServerCommand}"...\n`);
       close = () => killProcessTree(child.pid);
+
+      if (!patternMatch) {
+        process.stdout.write(`WARNING: Timed out waiting for the server to start listening.\n`);
+        process.stdout.write(`         Ensure the server prints "listening" when it is ready.\n`);
+        if (process.env.CI) process.stderr.write(`\nServer Output:\n${stdout}\n${stderr}\n`);
+      }
     }
 
     return {

packages/cli/src/collect/lighthouse-runner.js

@@ -5,6 +5,7 @@
  */
 'use strict';
 
+const os = require('os');
 const fs = require('fs');
 const path = require('path');
 const childProcess = require('child_process');
@@ -13,6 +14,13 @@ const {getSavedReportsDirectory} = require('@lhci/utils/src/saved-reports.js');
 const LH_CLI_PATH = path.join(require.resolve('lighthouse'), '../../lighthouse-cli/index.js');
 
 class LighthouseRunner {
+  /** @param {string} output */
+  static isOutputLhrLike(output) {
+    return (
+      output.startsWith('{') && output.includes('"lighthouseVersion":') && output.endsWith('}')
+    );
+  }
+
   /**
    * @param {string} url
    * @param {Partial<LHCI.CollectCommand.Options>} options
@@ -69,7 +77,7 @@ class LighthouseRunner {
     let stderr = '';
 
     const {args, cleanupFn} = LighthouseRunner.computeArgumentsAndCleanup(url, options);
-    const process = childProcess.spawn(LH_CLI_PATH, args);
+    const process = childProcess.spawn('node', [LH_CLI_PATH, ...args]);
 
     process.stdout.on('data', chunk => (stdout += chunk.toString()));
     process.stderr.on('data', chunk => (stderr += chunk.toString()));
@@ -78,6 +86,18 @@ class LighthouseRunner {
       cleanupFn();
       if (code === 0) return resolve(stdout);
 
+      // On Windows killing Chrome is extraordinarily flaky.
+      // If it looks like we got an LHR and we only died because of killing Chrome, ignore it.
+      if (
+        code === 1 &&
+        os.platform() === 'win32' &&
+        LighthouseRunner.isOutputLhrLike(stdout.trim()) &&
+        stderr.includes('Generating results...') &&
+        stderr.includes('Chrome could not be killed')
+      ) {
+        return resolve(stdout);
+      }
+
       /** @type {any} */
       const error = new Error(`Lighthouse failed with exit code ${code}`);
       error.stdout = stdout;

packages/cli/test/assert.test.js

@@ -7,6 +7,8 @@
 
 /* eslint-env jest */
 
+jest.retryTimes(3);
+
 const fs = require('fs');
 const path = require('path');
 const rimraf = require('rimraf');

packages/cli/test/autorun-start-server.test.js

@@ -5,6 +5,8 @@
  */
 'use strict';
 
+jest.retryTimes(3);
+
 /* eslint-env jest */
 
 const path = require('path');
@@ -24,9 +26,7 @@ describe('Lighthouse CI autorun CLI with startServerCommand', () => {
       {cwd: autorunDir}
     );
 
-    const stdoutClean = stdout.replace(/:\d{4,6}/g, ':XXXX').replace(/port \d{4,6}/, 'port XXXX');
-    const stderrClean = stderr.replace(/:\d{4,6}/g, ':XXXX').replace(/port \d{4,6}/, 'port XXXX');
-    expect(stdoutClean).toMatchInlineSnapshot(`
+    expect(stdout).toMatchInlineSnapshot(`
       "✅  .lighthouseci/ directory writable
       ⚠️   Configuration file found
       Healthcheck passed!
@@ -40,12 +40,12 @@ describe('Lighthouse CI autorun CLI with startServerCommand', () => {
 
       "
     `);
-    expect(stderrClean).toMatchInlineSnapshot(`
+    expect(stderr).toMatchInlineSnapshot(`
       "Checking assertions against 1 URL(s), 2 total run(s)
 
       1 result(s) for http://localhost:XXXX/
 
-        ✘  viewport failure for minScore assertion
+        X  viewport failure for minScore assertion
            Does not have a \`<meta name=\\"viewport\\">\` tag with \`width\` or \`initial-scale\`
            Documentation: https://web.dev/viewport
 

packages/cli/test/autorun-static-dir.test.js

@@ -5,6 +5,8 @@
  */
 'use strict';
 
+jest.retryTimes(3);
+
 /* eslint-env jest */
 
 const path = require('path');
@@ -19,11 +21,7 @@ describe('Lighthouse CI autorun CLI', () => {
       env: {LHCI_NO_LIGHTHOUSERC: undefined},
     });
 
-    const stdoutClean = stdout
-      .replace(/:\d{4,6}/g, ':XXXX')
-      .replace(/port \d{4,6}/, 'port XXXX')
-      .replace(/Open the report at.*\n/, 'Open the report at <link>\n');
-    const stderrClean = stderr.replace(/:\d{4,6}/g, ':XXXX').replace(/port \d{4,6}/, 'port XXXX');
+    const stdoutClean = stdout.replace(/Open the report at.*\n/, 'Open the report at <link>\n');
     expect(stdoutClean).toMatchInlineSnapshot(`
       "✅  .lighthouseci/ directory writable
       ✅  Configuration file found
@@ -46,12 +44,12 @@ describe('Lighthouse CI autorun CLI', () => {
 
       "
     `);
-    expect(stderrClean).toMatchInlineSnapshot(`
+    expect(stderr).toMatchInlineSnapshot(`
       "Checking assertions against 1 URL(s), 2 total run(s)
 
       1 result(s) for http://localhost:XXXX/good.html
 
-        ✘  viewport failure for minScore assertion
+        X  viewport failure for minScore assertion
            Does not have a \`<meta name=\\"viewport\\">\` tag with \`width\` or \`initial-scale\`
            Documentation: https://web.dev/viewport