Split out host tools into separate NPM

Summary:
This diff removes the host runtime from the `hermes-engine` NPM, and adds a separate `hermes-engine-cli` package with these plus additional host tools.

This reduces the installed size from 120MB to 18MB, and gives interested parties more host tools to play with.

Since we're renaming the executable, `test-e2e` fails. We'll have to push a new Hermes release and update the path in RN master.

Reviewed By: tmikov

Differential Revision: D20895983

fbshipit-source-id: 8777c8e84bfe6a6c12561dccd583eeba4b6ed3ff

.circleci/config.yml

@@ -357,7 +357,7 @@ jobs:
     docker:
       - image: ubuntu:19.04
     environment:
-      - yarn: yarnpkg
+      - YARN: yarnpkg
       - TERM: dumb
     steps:
       - run:
@@ -389,14 +389,15 @@ jobs:
           command: |
             cd npm
             cp /tmp/hermes/input/* .
-            $yarn install
-            $yarn pack
+            $YARN install
+            $YARN unpack-builds
+            $YARN create-npms
 
       - run:
           name: Copy artifacts
           command: |
             cd npm
-            cp hermes-engine-*.tgz /tmp/hermes/output
+            cp *.tgz /tmp/hermes/output
             # Also copy the other packages for the sole purpose of not having
             # to visit multiple jobs pages to download all release artifacts
             cp /tmp/hermes/input/*.tar.gz /tmp/hermes/output
@@ -476,7 +477,7 @@ jobs:
           name: Run E2E test with master Hermes and Release React Native
           command: |
             cd test/circleci-e2e
-            ./run.sh --hermes-npm-package /tmp/hermes/input/hermes-engine-*.tgz
+            ./run.sh --hermes-npm-package /tmp/hermes/input/hermes-engine-v*.tgz
           # The react-native setup tool takes AGES "Installing CocoaPods dependencies"
           no_output_timeout: 45m
       - store_artifacts:

CMakeLists.txt

@@ -598,11 +598,11 @@ add_custom_target(make_bundle_dir DEPENDS ${HERMES_GITHUB_BUNDLE_DIR})
 add_custom_command(
   OUTPUT ${HERMES_GITHUB_DIR}/${HERMES_CLI_GITHUB_FILE}
   WORKING_DIRECTORY ${HERMES_GITHUB_BUNDLE_DIR}
-  DEPENDS hermes hermes-repl make_bundle_dir
+  DEPENDS hermes hermesc hdb hbcdump hermes-repl make_bundle_dir
   VERBATIM
   COMMAND
     # We need bin/hermes or Release/bin/hermes.exe in a predictable location
-    ${CMAKE_COMMAND} -E copy $<TARGET_FILE:hermes> $<TARGET_FILE:hermes-repl> ${HERMES_GITHUB_EXTRAS} .
+    ${CMAKE_COMMAND} -E copy $<TARGET_FILE:hermes> $<TARGET_FILE:hermes-repl> $<TARGET_FILE:hermesc> $<TARGET_FILE:hdb> $<TARGET_FILE:hbcdump> ${HERMES_GITHUB_EXTRAS} .
   COMMAND
     ${CMAKE_COMMAND} -E tar zcf ${HERMES_GITHUB_DIR}/${HERMES_CLI_GITHUB_FILE} .
 )

npm/create-npms.js

@@ -0,0 +1,52 @@
+const shell = require('shelljs');
+const mainpkg = require('./package.json');
+const fs = require('fs');
+
+const commandLineArgs = require('command-line-args');
+const optionDefinitions = [
+  { name: 'dev', type: Boolean }
+];
+const options = commandLineArgs(optionDefinitions);
+
+// Abort script on any failure from any shelljs invocation
+shell.config.fatal = true;
+
+// shelljs has no mktemp, so kludge our own
+shell.mktemp = function() {
+  let dir = shell.tempdir() + "/" + (Date.now() + Math.random());
+  shell.mkdir(dir);
+  return dir;
+}
+
+function createNpm(dirname) {
+  if(!shell.test("-e", dirname + "/package.json")) {
+    throw new Error("Can't find package.json in " + dirname);
+  }
+  const here = shell.pwd();
+  const tmpdir = shell.mktemp("-d");
+
+  shell.cp("-R", "*", tmpdir);
+  shell.cp(dirname + "/*", tmpdir);
+  shell.cd(tmpdir);
+  shell.sed("-i", "%VERSION%", mainpkg.version, "package.json");
+  shell.exec("${YARN:-yarn} pack");
+  verifyManifest(require(`${here}/${dirname}/package.json`));
+  shell.cp(dirname + "*.tgz", here);
+  shell.cd(here);
+  shell.rm("-rf", tmpdir);
+}
+
+function verifyManifest(manifest) {
+  if (options.dev) return;
+
+  // TODO: Handle globs
+  const files = manifest.files.filter(s => s.indexOf("*") === -1);
+  for (const file of files) {
+    if (!fs.existsSync(file)) {
+      throw `File missing from manifest: ${file}`
+    }
+  }
+}
+
+const npms = shell.ls("*/package.json").map(c => c.replace(/\/.*/, ''));
+npms.forEach(createNpm);

npm/hermes-engine-cli/README.md

@@ -0,0 +1,13 @@
+## Hermes
+
+Hermes is a small and lightweight JavaScript VM optimized for running React
+Native apps on Android.  This package contains desktop binaries for testing and
+development purposes:
+
+* `hermes`, to run bytecode and run/compile source code
+* `hermes-repl`, a JavaScript REPL
+* `hdb`, a command line debugger
+* `hbcdump`, a low level bytecode inspection tool
+* `hermesc`, a subset of `hermes` that only compiles source code
+
+See [hermesengine.dev](https://hermesengine.dev) for more information.

npm/hermes-engine-cli/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "hermes-engine-cli",
+  "version": "%VERSION%",
+  "private": false,
+  "description": "Hermes Engine cli tools for host platforms",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:facebook/hermes.git"
+  },
+  "files": [
+    "README.md",
+    "linux64-bin/hbcdump",
+    "linux64-bin/hdb",
+    "linux64-bin/hermes",
+    "linux64-bin/hermes-repl",
+    "linux64-bin/hermesc",
+    "osx-bin/hbcdump",
+    "osx-bin/hdb",
+    "osx-bin/hermes",
+    "osx-bin/hermes-repl",
+    "osx-bin/hermesc",
+    "win64-bin/icudt64.dll",
+    "win64-bin/icuin64.dll",
+    "win64-bin/icuio64.dll",
+    "win64-bin/icutest64.dll",
+    "win64-bin/icutu64.dll",
+    "win64-bin/icuuc64.dll",
+    "win64-bin/msvcp140.dll",
+    "win64-bin/vcruntime140.dll",
+    "win64-bin/hbcdump.exe",
+    "win64-bin/hdb.exe",
+    "win64-bin/hermes-repl.exe",
+    "win64-bin/hermes.exe",
+    "win64-bin/hermesc.exe"
+  ]
+}

npm/hermes-engine/README.md

@@ -0,0 +1,13 @@
+## Hermes
+
+Hermes is a small and lightweight JavaScript VM optimized for running React
+Native apps on Android.
+
+See [hermesengine.dev](https://hermesengine.dev) for more information.
+
+This package contains a Hermes bytecode compiler for Windows, Linux and macOS,
+plus Android runtime libraries.
+
+Additional tools are available in
+[hermes-engine-cli](https://www.npmjs.com/package/hermes-engine-cli).
+

npm/hermes-engine/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "hermes-engine",
+  "version": "%VERSION%",
+  "private": false,
+  "description": "A JavaScript engine optimized for running React Native on Android",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:facebook/hermes.git"
+  },
+  "files": [
+    "README.md",
+    "android/hermes-debug.aar",
+    "android/hermes-release.aar",
+    "android/hermes-cppruntime-debug.aar",
+    "android/hermes-cppruntime-release.aar",
+    "android/include/**/*.h",
+    "linux64-bin/hermesc",
+    "osx-bin/hermesc",
+    "win64-bin/hermesc.exe",
+    "win64-bin/msvcp140.dll",
+    "win64-bin/vcruntime140.dll"
+  ]
+}

npm/package.json

@@ -1,35 +1,15 @@
 {
-  "name": "hermes-engine",
   "version": "0.4.0",
-  "private": false,
-  "description": "A JavaScript engine optimized for running React Native on Android",
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "git@github.com:facebook/hermes.git"
+  "scripts": {
+    "unpack-builds": "node unpack-builds.js",
+    "unpack-builds-dev": "node unpack-builds.js --dev",
+    "create-npms": "node create-npms.js",
+    "create-npms-dev": "node create-npms.js --dev"
   },
-  "files": [
-    "README.md",
-    "android/hermes-debug.aar",
-    "android/hermes-release.aar",
-    "android/hermes-cppruntime-debug.aar",
-    "android/hermes-cppruntime-release.aar",
-    "android/include/**/*.h",
-    "osx-bin/hermes",
-    "osx-bin/hermes-repl",
-    "linux64-bin/hermes",
-    "linux64-bin/hermes-repl",
-    "win64-bin/hermes.exe",
-    "win64-bin/hermes-repl.exe",
-    "win64-bin/*.dll"
-  ],
-  "devDependencies": {
+  "dependencies": {
+    "command-line-args": "^5.1.1",
     "request": "^2.88.0",
     "tar": "^4.4.8",
-    "command-line-args": "^5.1.1"
-  },
-  "scripts": {
-    "prepack": "node fetch.js",
-    "prepack-dev": "node fetch.js --dev"
+    "shelljs": "^0.8.3"
   }
 }

npm/fetch.js → npm/unpack-builds.js

@@ -116,20 +116,6 @@ async function unpackAll(files) {
     var names = inputs.map(c => c.name).join(", ");
     throw "No tarballs found. Please build or download at least one of: " + names;
   }
-
-  verifyManifest();
-}
-
-function verifyManifest() {
-  if (options.dev) return;
-
-  // TODO: Handle globs
-  const files = npmjson.files.filter(s => s.indexOf("*") === -1);
-  for (const file of files) {
-    if (!fs.existsSync(file)) {
-      throw `File missing from manifest: ${file}`
-    }
-  }
 }
 
 var inputs = [