Cleanup Build Scripts

.travis.yml

@@ -64,5 +64,3 @@ deploy:
   script: build-scripts/lerna-publish.js publish-travis --npm-client ${TRAVIS_BUILD_DIR}/build-scripts/npm-client.js --ignore-changes '**/*' --force-publish='*' --yes
   on:
     branch: master
-
-after_deploy: cat publish-log.txt

build-scripts/graal.js

@@ -41,9 +41,6 @@ process.on('unhandledRejection', error => {
   process.exit(1);
 });
 
-// As of the 20181028 release, the compiler is compatible with the latest RC of GRAAL.
-const GRAAL_BUILD_FROM_SRC = false;
-
 /**
  * Simple wrapper for the NodeJS spawn command to use promises
  *
@@ -92,143 +89,32 @@ const NATIVE_IMAGE_BUILD_ARGS = [
   '-H:IncludeResourceBundles=org.kohsuke.args4j.spi.Messages',
   '-H:IncludeResourceBundles=com.google.javascript.jscomp.parsing.ParserConfig',
   `-H:ReflectionConfigurationFiles=${path.resolve(__dirname, 'reflection-config.json')}`,
-  '-H:IncludeResources="(externs.zip)|(.*(js|txt))"',
+  '-H:IncludeResources=(externs.zip)|(.*(js|txt))',
   '-jar',
   path.resolve(process.cwd(), 'compiler.jar')
 ];
 let buildSteps = Promise.resolve();
-if (GRAAL_BUILD_FROM_SRC) {
-  const GRAAL_SRC_BASE = path.resolve(TEMP_PATH, 'graal');
-  const MX_SRC_BASE = path.resolve(TEMP_PATH, 'mx');
-
-  // The JVMCI version is strongly tied to the Graal version. When Graal is updated, the newest JVMCI release
-  // known to work should be used. This is often determined by trial and error.
-  const JVMCI_VERSION = 'jvmci-0.44';
-  const GRAAL_SOURCE_VERSION = 'vm-1.0.0-rc3';
-
-  // Clone the graal and mx repositories
-  buildSteps = buildSteps
-      .then(() => {
-        if (fs.existsSync(GRAAL_SRC_BASE)) {
-          return runCommand('git fetch', {cwd: GRAAL_SRC_BASE});
-        } else {
-          return runCommand('git clone https://github.com/oracle/graal.git', {cwd: TEMP_PATH});
-        }
-      })
-      .then(() => runCommand(`git checkout ${GRAAL_SOURCE_VERSION}`, {cwd: GRAAL_SRC_BASE}))
-      .then(() => {
-        if (!fs.existsSync(MX_SRC_BASE)) {
-          return runCommand('git clone https://github.com/graalvm/mx.git', {cwd: TEMP_PATH});
-        }
-      });
-
-  // Download the JDK with JVMCI support
-  let JDK_URL;
-  let JDK_FOLDER;
-  let JDK_PATH;
-  let JDK8_UPDATE_VERSION;
-  if (process.platform === 'darwin') {
-    // Custom built JDK8 with JVMCI support needed to build graal.
-    // Newer versions officially released at https://github.com/graalvm/openjdk8-jvmci-builder/releases
-    // were not compatible with the version of graal needed by the compiler.
-    // This specific version is known to work with the specific graal release we build.
-    // The JDK version is obtained from the JDK release url on GitHub.
-    JDK8_UPDATE_VERSION = '181';
-    JDK_URL =
-        'https://github.com/ChadKillingsworth/openjdk8-jvmci-builder/releases/download/' +
-        JVMCI_VERSION +
-        '/jdk1.8.0_' +
-        JDK8_UPDATE_VERSION +
-        '-' +
-        JVMCI_VERSION +
-        '-' +
-        process.platform +
-        '-amd64.tar.gz';
-
-    JDK_FOLDER = `jdk1.8.0_${JDK8_UPDATE_VERSION}-${JVMCI_VERSION}`;
-    JDK_PATH = path.resolve(TEMP_PATH, JDK_FOLDER, 'Contents', 'Home');
-  } else {
-    // Custom built JDK8 with JVMCI support needed to build graal.
-    // This specific version is known to work with the specific graal release we build.
-    // The JDK version is obtained from the JDK release url on GitHub.
-    JDK8_UPDATE_VERSION = '172';
-    JDK_URL =
-        'https://github.com/graalvm/openjdk8-jvmci-builder/releases/download/' +
-        JVMCI_VERSION +
-        '/openjdk-8u' +
-        JDK8_UPDATE_VERSION +
-        '-' +
-        JVMCI_VERSION +
-        '-' +
-        process.platform +
-        '-amd64.tar.gz';
-
-    JDK_FOLDER = `openjdk1.8.0_${JDK8_UPDATE_VERSION}-${JVMCI_VERSION}`;
-    JDK_PATH = path.resolve(TEMP_PATH, JDK_FOLDER);
-  }
+// Download Graal
+const GRAAL_ARCHIVE_FILE = `${GRAAL_FOLDER}.tar.gz`;
+if (!fs.existsSync(path.resolve(TEMP_PATH, GRAAL_FOLDER))) {
   buildSteps = buildSteps
       .then(() => {
-        if (!fs.existsSync(path.resolve(TEMP_PATH, 'jdk.tar.gz'))) {
+        // Download graal and extract the contents
+        if (!fs.existsSync(path.resolve(TEMP_PATH, GRAAL_ARCHIVE_FILE))) {
           return runCommand(
-              `curl --fail --show-error --location --progress-bar --output ${JDK_FOLDER}.tar.gz ${JDK_URL}`,
+              `curl --fail --show-error --location --progress-bar --output ${GRAAL_ARCHIVE_FILE} ${GRAAL_URL}`,
               {cwd: TEMP_PATH});
         }
       })
-      .then(() => runCommand(
-          `tar -xzf ${JDK_FOLDER}.tar.gz`,
-          {cwd: TEMP_PATH}));
-
-  // Build the native image tool then use it to build the compiler native image.
-  // Copy the resulting binary back to the package.
-  //
-  // At this point the temp folder should have:
-  //   - a JDK folder with JVMCI support
-  //   - an /mx folder with the mx build system
-  //   - a /graal folder checked out to the proper commit
-  buildSteps = buildSteps
-      .then(() => runCommand(
-          `${path.resolve(MX_SRC_BASE, 'mx')} -v build`,
-          {
-            cwd: path.resolve(GRAAL_SRC_BASE, 'substratevm'),
-            env: Object.assign({}, process.env, {JAVA_HOME: JDK_PATH, EXTRA_JAVA_HOMES: JDK_PATH})
-          }))
-      .then(() =>
-          // The mx launched version requires quotes around the arguments.
-          // But to correctly handle the quotes, we need to use a native shell to properly escape and pass them
-          // to the executable.
-          runCommand(`${path.resolve(MX_SRC_BASE, 'mx')} -v native-image ${NATIVE_IMAGE_BUILD_ARGS.join(' ')}`, {
-            cwd: path.resolve(GRAAL_SRC_BASE, 'substratevm'),
-            env: Object.assign({}, process.env, {JAVA_HOME: JDK_PATH, EXTRA_JAVA_HOMES: JDK_PATH}),
-            shell: true // Needed for proper quote escaping
-          }))
-      .then(() => new Promise((resolve, reject) =>
-          ncp(
-              path.resolve(GRAAL_SRC_BASE, 'substratevm', 'compiler'),
-              path.resolve(process.cwd(), 'compiler'),
-              err => err ? reject(err) : resolve())));
-} else {
-  // Download Graal
-  const GRAAL_ARCHIVE_FILE = `${GRAAL_FOLDER}.tar.gz`;
-  if (!fs.existsSync(path.resolve(TEMP_PATH, GRAAL_FOLDER))) {
-    buildSteps = buildSteps
-        .then(() => {
-          // Download graal and extract the contents
-          if (!fs.existsSync(path.resolve(TEMP_PATH, GRAAL_ARCHIVE_FILE))) {
-            return runCommand(
-                `curl --fail --show-error --location --progress-bar --output ${GRAAL_ARCHIVE_FILE} ${GRAAL_URL}`,
-                {cwd: TEMP_PATH});
-          }
-        })
-        .then(() => runCommand(`tar -xzf ${GRAAL_ARCHIVE_FILE}`, {cwd: TEMP_PATH}));
-  }
+      .then(() => runCommand(`tar -xzf ${GRAAL_ARCHIVE_FILE}`, {cwd: TEMP_PATH}));
+}
 
-  // Build the compiler native image.
-  const GRAAL_NATIVE_IMAGE_PATH = path.resolve(
-      TEMP_PATH,
-      `graalvm-ce-${GRAAL_VERSION}`,
-      ...(GRAAL_OS === 'macos' ? ['Contents', 'Home'] : []).concat(['bin', 'native-image']));
+// Build the compiler native image.
+const GRAAL_NATIVE_IMAGE_PATH = path.resolve(
+    TEMP_PATH,
+    `graalvm-ce-${GRAAL_VERSION}`,
+    ...(GRAAL_OS === 'macos' ? ['Contents', 'Home'] : []).concat(['bin', 'native-image']));
 
-  // Unlike the mx launched version, the native binary must not have quotes around arguments
-  buildSteps = buildSteps.then(
-      () => runCommand(`${GRAAL_NATIVE_IMAGE_PATH} ${NATIVE_IMAGE_BUILD_ARGS.join(' ').replace(/"/g, '')}`));
-}
+// Unlike the mx launched version, the native binary must not have quotes around arguments
+buildSteps = buildSteps.then(
+    () => runCommand(`${GRAAL_NATIVE_IMAGE_PATH} ${NATIVE_IMAGE_BUILD_ARGS.join(' ')}`));

build-scripts/lerna-publish.js

@@ -86,6 +86,23 @@ class TravisPublishCommand extends PublishCommand {
 
     return chain;
   }
+
+  /**
+   * After publication, write the custom npm client publish log to standard out
+   *
+   * @override
+   * @return {!Promise<undefined>}
+   */
+  execute() {
+    return super.execute().then(() => {
+      const logPath = path.resolve(__dirname, '..', 'publish-log.txt');
+      if (fs.existsSync(logPath)) {
+        process.stdout.write(fs.readFileSync(path.resolve(__dirname, '..', 'publish-log.txt'), 'utf8'));
+      } else {
+        process.stdout.write('publication log missing');
+      }
+    });
+  }
 }
 
 // New command meta data

build-scripts/npm-client.js

@@ -38,6 +38,8 @@ process.on('unhandledRejection', error => {
   process.exit(1);
 });
 
+const logMessageQueue = [];
+
 /**
  * Simple function to log out publication process to a file. Standard out doesn't work because lerna
  * swallows it.
@@ -94,7 +96,7 @@ function runCommand(cmd, args) {
 function checkThatVersionExists(packageWithVersion) {
   return runCommand('npm', ['view', packageWithVersion])
       .catch(results => {
-        logToFile(results.stderr);
+        logMessageQueue.push(results.stderr);
         return Promise.reject(new Error('version did not exist in registry'));
       });
 }
@@ -152,20 +154,20 @@ function npmPublish(packageInfo) {
       }
     }
   }).then(() => {
-    logToFile('    all dependencies published');
+    logMessageQueue.push('    all dependencies published');
     return runCommand('npm', process.argv.slice(2))
         .catch(results => {
           if (!/You cannot publish over the previously published versions/.test(results.stderr)) {
             return Promise.reject(new Error(`Publish failed ${JSON.stringify(results, null, 2)}`));
           }
         });
   }, err => {
-    logToFile(`  ❌ missing dependencies - ${err.message}`);
+    logMessageQueue.push(`  ❌ missing dependencies - ${err.message}`);
   });
 }
 
 // Log out what folder we are executing this from and the call arguments
-logToFile(`Publishing ${path.basename(process.cwd())}`);
+logMessageQueue.push(`Publishing ${path.basename(process.cwd())}`);
 
 // Add logic specific to each pacakge
 const pkg = require(path.resolve(process.cwd(), 'package.json'));
@@ -175,25 +177,33 @@ switch (pkg.name) {
     // We only want to publish the linux or osx package from a Travis instance running on the correct os
     if (pkg.name === 'google-closure-compiler-linux' && process.platform !== 'linux' ||
         pkg.name === 'google-closure-compiler-osx' && process.platform !== 'darwin') {
-      logToFile(`    skipping publication of ${pkg.name} - wrong platform`);
+      logMessageQueue.push(`    skipping publication of ${pkg.name} - wrong platform`);
       process.exitCode = 0;
     } else {
       npmPublish(pkg)
-          .then(() => logToFile('  ✅ publish succeeded'))
+          .then(() => {
+            logMessageQueue.push('  ✅ publish succeeded');
+            logToFile(logMessageQueue.join('\n'));
+          })
           .catch(err => {
             process.exitCode = 1;
-            logToFile('   ❌ publish failed');
+            logMessageQueue.push('   ❌ publish failed');
+            logToFile(logMessageQueue.join('\n'));
             return Promise.reject(err);
           });
     }
     break;
 
   default:
     npmPublish(pkg)
-        .then(() => logToFile('  ✅ publish succeeded'))
+        .then(() => {
+          logMessageQueue.push('  ✅ publish succeeded');
+          logToFile(logMessageQueue.join('\n'));
+        })
         .catch(err => {
           process.exitCode = 1;
-          logToFile('   ❌ publish failed');
+          logMessageQueue.push('   ❌ publish failed');
+          logToFile(logMessageQueue.join('\n'));
           return Promise.reject(err);
         });
     break;

deployments.md

@@ -2,6 +2,8 @@
 
 *You now need yarn installed: https://yarnpkg.com/en/docs/install*
 
+## Deploying new releases of the main compiler
+
  1. Update the compiler submodule pointer to the tagged release.
      * if not already done, update the compiler submodule
        `git submodule init && git submodule update`
@@ -13,3 +15,30 @@
     The `COMPILER_VERSION_NUMBER` should not include the preceding `v` - example: 20181008.
     The command will ask you to verify that you wish to create new versions for each package.
     Once confirmed, version numbers will be committed, the commit tagged and changes pushed.
+
+## Deploying changes to the package CLIs or plugins
+
+Features and fixes to the packages in this repo need not wait for a main compiler release.
+They can be published at any time if they are backwards compatible with the last major version.
+Breaking changes should be deployed at the same time as a major compiler release.
+
+ 1. Run `yarn install`.
+ 2. Run `node_modules/.bin/lerna version`.
+    The command will ask you to choose a new version number.
+    Patch level versions should be used for fixing issues.
+    Minor level versions should be used for new features that are backwards compatible.
+
+## Verifying Publication Was Successful
+
+The `lerna version` command will push a new tagged commit. Travis will automatically start building
+this commit. Expand the `Deploying application` section at the bottom of the log. For each
+successfully published package, a `✅ publish succeeded` line should be present. In addition,
+the npm registry page for each package should list the newly published version.
+
+Since the native images can only be build on their respective OS, whichever Travis build
+finishes last will publish the main package as it waits until all dependent packages have
+been published.
+
+*Note: The lerna log file lines "lerna success published 5 packages" is misleading. Lerna will
+succeed even if the package has already been published or failed to publish due to missing
+dependencies.*