Merge branch 'master' into lazy-third-party

.github/workflows/ci.yml

@@ -29,8 +29,6 @@ jobs:
     - run: sudo apt-get install xvfb
     - name: yarn test-clients
       run: xvfb-run --auto-servernum yarn test-clients
-    - name: yarn test-bundle
-      run: xvfb-run --auto-servernum yarn test-bundle
     - name: yarn test-docs
       run: xvfb-run --auto-servernum yarn test-docs
     - name: yarn test-viewer
@@ -44,6 +42,9 @@ jobs:
     - run: yarn i18n:checks
     - run: yarn dogfood-lhci
 
+    # Fail if any changes were written to any source files or generated untracked files (ex, from: build/build-cdt-lib.js).
+    - run: git add -A && git diff --cached --exit-code
+
     # buildtracker runs `git merge-base HEAD origin/master` which needs more history than depth=1. https://github.com/paularmstrong/build-tracker/issues/106
     - name: Deepen git fetch (for buildtracker)
       run: git fetch --deepen=100
@@ -60,9 +61,6 @@ jobs:
         name: dist
         path: dist/
 
-    # Fail if any changes were written to source files (ex, from: build/build-cdt-lib.js).
-    - run: git diff --exit-code
-
   # `unit` includes just unit and proto tests.
   unit:
     strategy:
@@ -113,6 +111,9 @@ jobs:
     - run: yarn diff:sample-json
       if: matrix.os == 'windows-latest'
 
+    # Fail if any changes were written to any source files or generated untracked files (ex, from -GA).
+    - run: git add -A && git diff --cached --exit-code
+
   # `smoke` runs as a matrix across 4 jobs:
   #  * The smoketest groups are split across two runners, to parallelize.
   #  * Then, those are run with both Chrome stable and ToT Chromium, in parallel
@@ -154,6 +155,9 @@ jobs:
     - name: Run smoke tests
       run: xvfb-run --auto-servernum yarn smoke --debug -j=1 --retries=2 --invert-match ${{ matrix.smoke-test-invert }} $SMOKE_GROUP_1
 
+    # Fail if any changes were written to source files.
+    - run: git diff --exit-code
+
   # Only run smoke tests for windows against stable chrome.
   smoke-windows:
     runs-on: windows-latest
@@ -172,3 +176,26 @@ jobs:
 
     - name: Run smoke tests
       run: yarn smoke --debug -j=1 --retries=2 dbw oopif offline lantern metrics
+
+  smoke-bundle:
+    runs-on: ubuntu-latest
+    name: smoke_bundle
+
+    steps:
+    - name: git clone
+      uses: actions/checkout@v2
+
+    - name: Use Node.js 10.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 10.x
+
+    - run: yarn --frozen-lockfile
+    - run: yarn build-devtools
+
+    - run: sudo apt-get install xvfb
+    - name: yarn test-bundle
+      run: xvfb-run --auto-servernum yarn test-bundle
+
+    # Fail if any changes were written to source files.
+    - run: git diff --exit-code

.github/workflows/devtools.yml

@@ -33,7 +33,7 @@ jobs:
           ${{ env.DEVTOOLS_PATH }}
           ${{ env.BLINK_TOOLS_PATH }}
           ${{ github.workspace }}/lighthouse/.tmp/chromium-web-tests/content-shells
-        key: ${{ runner.os }}-${{ hashFiles('lighthouse/.github/workflows/devtools.yml', 'lighthouse/lighthouse-core/test/chromium-web-tests/download-*', 'lighthouse/clients/devtools-entry.js', 'lighthouse/clients/devtools-report-assets.js', 'lighthouse/build/build-bundle.js', 'lighthouse/build/build-dt-report-resources.js') }}
+        key: ${{ runner.os }}-${{ hashFiles('lighthouse/.github/workflows/devtools.yml', 'lighthouse/lighthouse-core/test/chromium-web-tests/*', 'lighthouse/clients/devtools-entry.js', 'lighthouse/clients/devtools-report-assets.js', 'lighthouse/build/build-bundle.js', 'lighthouse/build/build-dt-report-resources.js') }}
 
     - name: Use Node.js 10.x
       uses: actions/setup-node@v1

.npmignore

@@ -31,6 +31,8 @@ timings-data/
 # ignore all folders named as such
 node_modules
 latest-run
+chromium-webtests
+.tmp
 
 # generated files
 **/pages/scripts/lighthouse-report.js

build-tracker.config.js

@@ -8,7 +8,7 @@ module.exports = {
     'dist/lightrider/lighthouse-lr-bundle.js',
     'dist/extension/scripts/lighthouse-ext-bundle.js',
     'dist/lighthouse-dt-bundle.js',
-    'dist/viewer/src/viewer.js',
+    'dist/gh-pages/viewer/src/bundled.js',
     'dist/lightrider/report-generator-bundle.js',
     'dist/dt-report-resources/report.js',
     'dist/dt-report-resources/report-generator.js',

build/build-viewer.js

@@ -6,106 +6,17 @@
 'use strict';
 
 const fs = require('fs');
-const path = require('path');
-const {promisify} = require('util');
-const readFileAsync = promisify(fs.readFile);
-const writeFileAsync = promisify(fs.writeFile);
-const mkdir = fs.promises.mkdir;
-
 const browserify = require('browserify');
-const cpy = require('cpy');
-const ghPages = require('gh-pages');
-const glob = promisify(require('glob'));
-const lighthousePackage = require('../package.json');
-const rimraf = require('rimraf');
-const terser = require('terser');
+const GhPagesApp = require('./gh-pages-app.js');
 const {minifyFileTransform} = require('./build-utils.js');
-
 const htmlReportAssets = require('../lighthouse-core/report/html/html-report-assets.js');
-const sourceDir = `${__dirname}/../lighthouse-viewer`;
-const distDir = `${__dirname}/../dist/viewer`;
-
-const license = `/*
-* @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-* or implied. See the License for the specific language governing
-* permissions and limitations under the License.
-*/`;
-
-/**
- * Evaluates path glob and loads all identified files as an array of strings.
- * @param {string} pattern
- * @return {Promise<Array<string>>}
- */
-async function loadFiles(pattern) {
-  const filePaths = await glob(pattern);
-  return Promise.all(filePaths.map(path => readFileAsync(path, {encoding: 'utf8'})));
-}
-
-/**
- * Write a file to filePath, creating parent directories if needed.
- * @param {string} filePath
- * @param {string} data
- * @return {Promise<void>}
- */
-async function safeWriteFileAsync(filePath, data) {
-  const fileDir = path.dirname(filePath);
-  await mkdir(fileDir, {recursive: true});
-  return writeFileAsync(filePath, data);
-}
-
-/**
- * Copy static assets.
- * @return {Promise<void>}
- */
-function copyAssets() {
-  return cpy([
-    'images/**/*',
-    'sw.js',
-    'manifest.json',
-  ], distDir, {
-    cwd: `${sourceDir}/app/`,
-    parents: true,
-  });
-}
-
-/**
- * Concat report and viewer stylesheets into single viewer.css file.
- * @return {Promise<void>}
- */
-async function css() {
-  const reportCss = htmlReportAssets.REPORT_CSS;
-  const viewerCss = await readFileAsync(`${sourceDir}/app/styles/viewer.css`, {encoding: 'utf8'});
-  await safeWriteFileAsync(`${distDir}/styles/viewer.css`, [reportCss, viewerCss].join('\n'));
-}
-
-/**
- * Insert report templates into html and copy to dist.
- * @return {Promise<void>}
- */
-async function html() {
-  let htmlSrc = await readFileAsync(`${sourceDir}/app/index.html`, {encoding: 'utf8'});
-  htmlSrc = htmlSrc.replace(/%%LIGHTHOUSE_TEMPLATES%%/, htmlReportAssets.REPORT_TEMPLATES);
-
-  await safeWriteFileAsync(`${distDir}/index.html`, htmlSrc);
-}
 
 /**
- * Combine multiple JS files into single viewer.js file.
- * @return {Promise<void>}
+ * Build viewer, optionally deploying to gh-pages if `--deploy` flag was set.
  */
-async function compileJs() {
+async function run() {
   // JS bundle from browserified ReportGenerator.
-  const generatorFilename = `${sourceDir}/../lighthouse-core/report/report-generator.js`;
+  const generatorFilename = `${__dirname}/../lighthouse-core/report/report-generator.js`;
   const generatorBrowserify = browserify(generatorFilename, {standalone: 'ReportGenerator'})
     .transform('@wardpeet/brfs', {
       readFileSyncTransform: minifyFileTransform,
@@ -118,73 +29,35 @@ async function compileJs() {
       resolve(src.toString());
     });
   });
-  const generatorJs = await generatorJsPromise;
-
-  // Report renderer scripts.
-  const rendererJs = htmlReportAssets.REPORT_JAVASCRIPT;
-
-  // idb-keyval dependency.
-  const idbKeyvalPath = require.resolve('idb-keyval/dist/idb-keyval-min.js');
-  const idbKeyvalJs = await readFileAsync(idbKeyvalPath, 'utf8');
-
-  // Current Lighthouse version as a global variable.
-  const versionJs = `window.LH_CURRENT_VERSION = '${lighthousePackage.version}';`;
-
-  // Viewer-specific JS files.
-  const viewJsFiles = await loadFiles(`${sourceDir}/app/src/*.js`);
-
-  const contents = [
-    `"use strict";`,
-    generatorJs,
-    rendererJs,
-    idbKeyvalJs,
-    versionJs,
-    ...viewJsFiles,
-  ];
-  const options = {
-    output: {preamble: license}, // Insert license at top.
-  };
-  const uglified = terser.minify(contents, options);
-  if (uglified.error || !uglified.code) {
-    throw uglified.error;
-  }
 
-  await safeWriteFileAsync(`${distDir}/src/viewer.js`, uglified.code);
-}
-
-/**
- * Publish viewer to gh-pages branch.
- * @return {Promise<void>}
- */
-async function deploy() {
-  return new Promise((resolve, reject) => {
-    ghPages.publish(distDir, {
-      add: true, // keep existing files
-      dest: 'viewer',
-      message: `Update viewer to lighthouse@${lighthousePackage.version}`,
-    }, err => {
-      if (err) return reject(err);
-      resolve();
-    });
+  const app = new GhPagesApp({
+    name: 'viewer',
+    appDir: `${__dirname}/../lighthouse-viewer/app`,
+    html: {path: 'index.html'},
+    htmlReplacements: {
+      '%%LIGHTHOUSE_TEMPLATES%%': htmlReportAssets.REPORT_TEMPLATES,
+    },
+    stylesheets: [
+      htmlReportAssets.REPORT_CSS,
+      {path: 'styles/*'},
+    ],
+    javascripts: [
+      await generatorJsPromise,
+      htmlReportAssets.REPORT_JAVASCRIPT,
+      fs.readFileSync(require.resolve('idb-keyval/dist/idb-keyval-min.js'), 'utf8'),
+      {path: 'src/*'},
+    ],
+    assets: [
+      {path: 'images/**/*'},
+      {path: 'manifest.json'},
+    ],
   });
-}
 
-/**
- * Build viewer, optionally deploying to gh-pages if `--deploy` flag was set.
- */
-async function run() {
-  // Clean and build.
-  rimraf.sync(distDir);
-  await Promise.all([
-    compileJs(),
-    html(),
-    css(),
-    copyAssets(),
-  ]);
+  await app.build();
 
   const argv = process.argv.slice(2);
   if (argv.includes('--deploy')) {
-    await deploy();
+    await app.deploy();
   }
 }