🏗🚀✨ Significantly speed up gulp dist and gulp check-types (#21939)

build-system/runner/build.xml

@@ -22,7 +22,6 @@
   <property name="src.dir" value="${basedir}/src"/>
   <property name="test.dir" value="${basedir}/test"/>
   <property name="build.dir" value="${basedir}/build"/>
-  <property name="lib.dir" value="${basedir}/lib"/>
   <property name="dist.dir" value="${basedir}/dist"/>
   <property name="runner-jarfile" value="${build.dir}/${ant.project.name}.jar"/>
   <property name="compiler.dir" value="${basedir}/../../third_party/closure-compiler"/>
@@ -32,16 +31,13 @@
 
   <target name="jar" depends="compile">
     <jar destfile="${runner-jarfile}">
-      <zipfileset src="${lib.dir}/jar-in-jar-loader.zip"/>
       <fileset dir="${classes.dir}"/>
-      <fileset dir="${compiler.dir}">
+      <zipgroupfileset dir="${compiler.dir}">
         <include name="compiler.jar"/>
-      </fileset>
+      </zipgroupfileset>
       <manifest>
-        <attribute name="Main-Class" value="org.eclipse.jdt.internal.jarinjarloader.JarRsrcLoader"/>
-        <attribute name="Rsrc-Main-Class" value="org.ampproject.AmpCommandLineRunner"/>
-        <attribute name="Class-Path" value="."/>
-        <attribute name="Rsrc-Class-Path" value="./ compiler.jar"/>
+        <attribute name="Main-Class" value="org.ampproject.AmpCommandLineRunner"/>
+        <attribute name="Class-Path" value="./ compiler.jar"/>
       </manifest>
     </jar>
     <mkdir dir="${dist.dir}"/>

build-system/tasks/compile.js

@@ -21,20 +21,20 @@ const colors = require('ansi-colors');
 const fs = require('fs-extra');
 const gulp = require('gulp');
 const nop = require('gulp-nop');
-const {isTravisBuild} = require('../travis');
-const {VERSION: internalRuntimeVersion} = require('../internal-version') ;
-
 const rename = require('gulp-rename');
 const rimraf = require('rimraf');
 const shortenLicense = require('../shorten-license');
 const sourcemaps = require('gulp-sourcemaps');
 const {highlight} = require('cli-highlight');
+const {isTravisBuild} = require('../travis');
 const {singlePassCompile} = require('../single-pass');
+const {VERSION: internalRuntimeVersion} = require('../internal-version') ;
 
 const isProdBuild = !!argv.type;
 const queue = [];
 let inProgress = 0;
 const MAX_PARALLEL_CLOSURE_INVOCATIONS = 4;
+const NAILGUN_PORT = '2113'; // Also used in gulpfile.js
 
 // Compiles AMP with the closure compiler. This is intended only for
 // production use. During development we intend to continue using
@@ -343,7 +343,7 @@ function compile(entryModuleFilenames, outputDir, outputFilename, options) {
     externs.push('build-system/amp.multipass.extern.js');
 
     /* eslint "google-camelcase/google-camelcase": 0*/
-    const compilerOptions = {
+    let compilerOptions = {
       compilation_level: options.compilationLevel || 'SIMPLE_OPTIMIZATIONS',
       // Turns on more optimizations.
       assume_function_wrapper: true,
@@ -404,19 +404,49 @@ function compile(entryModuleFilenames, outputDir, outputFilename, options) {
 
     let compilerErrors = '';
     const pluginOptions = {
-      platform: ['java'], // Override the JAR used by closure compiler
+      platform: ['java'], // Override the binary used by closure compiler
       extraArguments: ['-XX:+TieredCompilation'], // Significant speed up!
       logger: errors => compilerErrors = errors, // Capture compiler errors
     };
 
+    // SIGNIFICANTLY speed up compilation on Mac and Linux using nailgun
+    // See https://github.com/facebook/nailgun.
+    if (!argv.single_pass &&
+        (process.platform == 'darwin' || process.platform == 'linux')) {
+      const compilerOptionsArray = [
+        '--nailgun-port',
+        NAILGUN_PORT,
+        'org.ampproject.AmpCommandLineRunner',
+        '--',
+      ];
+      Object.keys(compilerOptions).forEach(function(option) {
+        const value = compilerOptions[option];
+        if (value instanceof Array) {
+          value.forEach(function(item) {
+            compilerOptionsArray.push('--' + option + '=' + item);
+          });
+        } else {
+          if (value != null) {
+            compilerOptionsArray.push('--' + option + '=' + value);
+          } else {
+            compilerOptionsArray.push('--' + option);
+          }
+        }
+      });
+      compilerOptions = compilerOptionsArray; // nailgun-runner takes an array
+      pluginOptions.platform = ['native']; // nailgun-runner isn't a java binary
+      pluginOptions.extraArguments = null; // Already part of nailgun-server
+    }
+
     // Override to local closure compiler JAR
     closureCompiler.compiler.JAR_PATH =
         require.resolve('../runner/dist/runner.jar');
 
-    const handleCompilerError = function() {
+    const handleCompilerError = function(err) {
       console./*OK*/error(colors.red(
           'Compilation failed for ' + outputFilename + ':\n') +
-          formatClosureCompilerError(compilerErrors));
+          formatClosureCompilerError(compilerErrors) + '\n' +
+          err);
       process.exit(1);
     };
 

gulpfile.js

@@ -31,6 +31,7 @@ const minimatch = require('minimatch');
 const minimist = require('minimist');
 const path = require('path');
 const rimraf = require('rimraf');
+const sleep = require('sleep-promise');
 const source = require('vinyl-source-stream');
 const touch = require('touch');
 const watchify = require('watchify');
@@ -40,6 +41,7 @@ const {applyConfig, removeConfig} = require('./build-system/tasks/prepend-global
 const {cleanupBuildDir, closureCompile} = require('./build-system/tasks/compile');
 const {createCtrlcHandler, exitCtrlcHandler} = require('./build-system/ctrlcHandler');
 const {createModuleCompatibleES5Bundle} = require('./build-system/tasks/create-module-compatible-es5-bundle');
+const {exec, execScriptAsync, getStdout} = require('./build-system/exec');
 const {isTravisBuild} = require('./build-system/travis');
 const {jsifyCssAsync} = require('./build-system/tasks/jsify-css');
 const {serve} = require('./build-system/tasks/serve.js');
@@ -65,7 +67,7 @@ let extensionsToBuild = null;
 // All a4a extensions.
 const adVendors = [];
 
-const {green, red, cyan} = colors;
+const {green, red, cyan, yellow} = colors;
 
 // Minified targets to which AMP_CONFIG must be written.
 const minifiedRuntimeTarget = 'dist/v0.js';
@@ -84,6 +86,16 @@ const unminified3pTarget = 'dist.3p/current/integration.js';
 
 const maybeUpdatePackages = isTravisBuild() ? [] : ['update-packages'];
 
+// Used to start and stop the Closure nailgun server
+let nailgunRunnerReplacer;
+const nailgunRunner =
+    require.resolve('./third_party/nailgun/nailgun-runner');
+const nailgunServer =
+    require.resolve('./third_party/nailgun/nailgun-server.jar');
+const customRunner = require.resolve('./build-system/runner/dist/runner.jar');
+const NAILGUN_PORT = '2113'; // Also used in build-system/tasks/compile.js
+const NAILGUN_STARTUP_TIMEOUT_MS = 5 * 1000;
+
 /**
  * Tasks that should print the `--nobuild` help text.
  * @private @const {!Set<string>}
@@ -915,6 +927,85 @@ function build() {
   return performBuild().then(() => exitCtrlcHandler(handlerProcess));
 }
 
+/**
+ * Starts a nailgun server (provides a fast-running closure compiler instance),
+ * and replaces google-closure-compiler's binary with the nailgun runner
+ */
+async function startNailgunServer() {
+  if (argv.single_pass) {
+    return;
+  }
+  // Replace default binary with nailgun on linux and macos
+  if (process.platform == 'darwin') {
+    nailgunRunnerReplacer = require('require-hijack')
+        .replace('google-closure-compiler-osx').with(nailgunRunner);
+  } else if (process.platform == 'linux') {
+    nailgunRunnerReplacer = require('require-hijack')
+        .replace('google-closure-compiler-linux').with(nailgunRunner);
+  } else {
+    log(yellow('WARNING:'), 'Cannot run', cyan('nailgun-server.jar'),
+        'on', cyan(process.platform));
+    log(yellow('WARNING:'),
+        'Closure compiler will be significantly slower than on',
+        cyan('macos'), 'or', cyan('linux'));
+    return;
+  }
+
+  // Start up the nailgun server after cleaning up old instances (if any)
+  const startNailgunServerCmd =
+      'java -XX:+TieredCompilation -server -cp ' +
+      `${nailgunServer}:${customRunner} ` +
+      `com.facebook.nailgun.NGServer ${NAILGUN_PORT}`;
+  const stopNailgunServerCmd =
+      `${nailgunRunner} --nailgun-port ${NAILGUN_PORT} ng-stop`;
+  const getVersionCmd =
+      `${nailgunRunner} --nailgun-port ${NAILGUN_PORT} ` +
+      'org.ampproject.AmpCommandLineRunner -- --version';
+  if (!isTravisBuild()) {
+    log('Starting', cyan('nailgun-server.jar'), 'on port', cyan(NAILGUN_PORT));
+  }
+  exec(stopNailgunServerCmd, {stdio: 'pipe'});
+  execScriptAsync(startNailgunServerCmd, {stdio: 'pipe'});
+
+  // Ensure that the nailgun server is up and running
+  const end = Date.now() + NAILGUN_STARTUP_TIMEOUT_MS;
+  while (Date.now() < end) {
+    try {
+      const version = getStdout(getVersionCmd).trim();
+      if (/Version/.test(version)) {
+        return;
+      }
+    } catch (e) {
+      await sleep(1000);
+    }
+  }
+  log(red('ERROR:'), 'Could not start',
+      cyan(nailgunServer), 'on port', cyan(NAILGUN_PORT) + '...');
+  process.exit(1);
+}
+
+/**
+ * Stops the nailgun server if it's running, and restores the binary used by
+ * google-closure-compiler
+ */
+async function stopNailgunServer() {
+  if (argv.single_pass) {
+    return;
+  }
+  if (nailgunRunnerReplacer) {
+    nailgunRunnerReplacer.restore();
+  }
+  if (process.platform == 'darwin' || process.platform == 'linux') {
+    const stopNailgunServerCmd =
+        `${nailgunRunner} --nailgun-port ${NAILGUN_PORT} ng-stop`;
+    if (!isTravisBuild()) {
+      log('Stopping', cyan('nailgun-server.jar'), 'on port',
+          cyan(NAILGUN_PORT));
+    }
+    exec(stopNailgunServerCmd, {stdio: 'pipe'});
+  }
+}
+
 /**
  * Dist Build
  * @return {!Promise}
@@ -932,13 +1023,16 @@ function dist() {
   }
   if (argv.single_pass) {
     if (!isTravisBuild()) {
-      log(green('Not building any AMP extensions in'), cyan('single_pass'),
+      log(green('Building all AMP extensions in'), cyan('single_pass'),
           green('mode.'));
     }
   } else {
     parseExtensionFlags();
   }
   return compileCss(/* watch */ undefined, /* opt_compileAll */ true)
+      .then(async() => {
+        await startNailgunServer();
+      })
       .then(() => {
         return Promise.all([
           compile(false, true, true),
@@ -964,6 +1058,8 @@ function dist() {
           // New line after all the compilation progress dots on Travis.
           console.log('\n');
         }
+      }).then(async() => {
+        await stopNailgunServer();
       }).then(() => {
         return copyAliasExtensions();
       }).then(() => {
@@ -1056,44 +1152,50 @@ function checkTypes() {
     return './extensions/' + extension.name + '/' +
         extension.version + '/' + extension.name + '.js';
   }).sort();
-  return compileCss().then(() => {
-    return Promise.all([
-      closureCompile(compileSrcs.concat(extensionSrcs), './dist',
-          'check-types.js', {
-            include3pDirectories: true,
-            includePolyfills: true,
-            extraGlobs: ['src/inabox/*.js'],
-            typeCheckOnly: true,
-          }),
-      // Type check 3p/ads code.
-      closureCompile(['./3p/integration.js'], './dist',
-          'integration-check-types.js', {
-            externs: ['ads/ads.extern.js'],
-            include3pDirectories: true,
-            includePolyfills: true,
-            typeCheckOnly: true,
-          }),
-      closureCompile(['./3p/ampcontext-lib.js'], './dist',
-          'ampcontext-check-types.js', {
-            externs: ['ads/ads.extern.js'],
-            include3pDirectories: true,
-            includePolyfills: true,
-            typeCheckOnly: true,
-          }),
-      closureCompile(['./3p/iframe-transport-client-lib.js'], './dist',
-          'iframe-transport-client-check-types.js', {
-            externs: ['ads/ads.extern.js'],
-            include3pDirectories: true,
-            includePolyfills: true,
-            typeCheckOnly: true,
-          }),
-    ]);
-  }).then(() => {
-    if (isTravisBuild()) {
-      // New line after all the compilation progress dots on Travis.
-      console.log('\n');
-    }
-  }).then(() => exitCtrlcHandler(handlerProcess));
+  return compileCss()
+      .then(async() => {
+        await startNailgunServer();
+      })
+      .then(() => {
+        return Promise.all([
+          closureCompile(compileSrcs.concat(extensionSrcs), './dist',
+              'check-types.js', {
+                include3pDirectories: true,
+                includePolyfills: true,
+                extraGlobs: ['src/inabox/*.js'],
+                typeCheckOnly: true,
+              }),
+          // Type check 3p/ads code.
+          closureCompile(['./3p/integration.js'], './dist',
+              'integration-check-types.js', {
+                externs: ['ads/ads.extern.js'],
+                include3pDirectories: true,
+                includePolyfills: true,
+                typeCheckOnly: true,
+              }),
+          closureCompile(['./3p/ampcontext-lib.js'], './dist',
+              'ampcontext-check-types.js', {
+                externs: ['ads/ads.extern.js'],
+                include3pDirectories: true,
+                includePolyfills: true,
+                typeCheckOnly: true,
+              }),
+          closureCompile(['./3p/iframe-transport-client-lib.js'], './dist',
+              'iframe-transport-client-check-types.js', {
+                externs: ['ads/ads.extern.js'],
+                include3pDirectories: true,
+                includePolyfills: true,
+                typeCheckOnly: true,
+              }),
+        ]);
+      }).then(() => {
+        if (isTravisBuild()) {
+          // New line after all the compilation progress dots on Travis.
+          console.log('\n');
+        }
+      }).then(async() => {
+        await stopNailgunServer();
+      }).then(() => exitCtrlcHandler(handlerProcess));
 }
 
 /**

package.json

@@ -160,6 +160,7 @@
     "regexp.escape": "1.0.2",
     "request": "2.88.0",
     "request-promise": "4.2.4",
+    "require-hijack": "1.2.1",
     "rimraf": "2.6.3",
     "rocambole": "0.7.0",
     "sinon": "7.3.2",

third_party/nailgun/README.amp

@@ -0,0 +1,15 @@
+URL: https://github.com/facebook/nailgun
+License: Apache license, version 2.0
+License File: https://github.com/facebook/nailgun/blob/master/LICENSE.txt
+
+Description:
+Nailgun is a client, protocol, and server for running Java programs from the command line without incurring the JVM startup overhead.
+
+The AMP Project uses a custom version of closure compiler available at build-system/runner/dist/runner.jar. The compiler binary is launched several dozen times during the default multi-pass minification process (gulp dist). In order to speed this up, we use nailgun to launch a reusable instance of closure compiler.
+
+Files in this directory (copied from https://github.com/facebook/nailgun/releases/tag/nailgun-all-v1.0.0):
+- nailgun-server.jar (copied from nailgun-server-1.0.0-SNAPSHOT.jar)
+- nailgun-runner (copied from ng.py)
+
+Local Modifications:
+None.