One way of cross-compiling local binary packages

If $METEOR_BINARY_DEP_WORKAROUND is set, then when bundling for a
non-host platform (build/bundle/deploy commands only), if a package has
no server unibuild for the target architecture, use the host
architecture and replace the npm modules with a package.json and
npm-shrinkwrap.json.  Also write out a top-level setup.sh script (inside
programs/server) which runs npm install in all such directories.

To support this, we make sure to save the package.json and
npm-shrinkwrap.json files in various intermediate directories in case we
need them later.  (We put them inside node_modules because that is what
gets copied from source tree to isopack.)

tools/cli/commands.js

@@ -925,7 +925,8 @@ on an OS X system.");
       //     packages with binary npm dependencies
       serverArch: bundleArch,
       buildMode: options.debug ? 'development' : 'production',
-    }
+    },
+    providePackageJSONForUnavailableBinaryDeps: !!process.env.METEOR_BINARY_DEP_WORKAROUND,
   });
   if (bundleResult.errors) {
     Console.error("Errors prevented bundling:");

tools/isobuild/bundler.js

@@ -223,6 +223,9 @@ var NodeModulesDirectory = function (options) {
 
   // Optionally, files to discard.
   self.npmDiscards = options.npmDiscards;
+
+  // Write a package.json file instead of copying the full directory.
+  self.writePackageJSON = !!options.writePackageJSON;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -415,7 +418,9 @@ class Target {
     //  and prodOnly packages are included; defaults to 'production'
     buildMode,
     // directory on disk where to store the cache for things like linker
-    bundlerCacheDir
+    bundlerCacheDir,
+    // whether to substitute a package.json for unavailable binary deps
+    providePackageJSONForUnavailableBinaryDeps,
     // ... see subclasses for additional options
   }) {
     this.packageMap = packageMap;
@@ -462,6 +467,9 @@ class Target {
     this.buildMode = buildMode || 'production';
 
     this.bundlerCacheDir = bundlerCacheDir;
+
+    this.providePackageJSONForUnavailableBinaryDeps
+      = providePackageJSONForUnavailableBinaryDeps;
   }
 
   // Top-level entry point for building a target. Generally to build a
@@ -561,7 +569,9 @@ class Target {
         if (p.prodOnly && this.buildMode !== 'production') {
           return;
         }
-        const unibuild = p.getUnibuildAtArch(this.arch);
+        const unibuild = p.getUnibuildAtArch(this.arch, {
+          allowWrongPlatform: this.providePackageJSONForUnavailableBinaryDeps
+        });
         unibuild && rootUnibuilds.push(unibuild);
       });
 
@@ -592,7 +602,8 @@ class Target {
           arch: this.arch,
           isopackCache: isopackCache,
           skipDebugOnly: this.buildMode !== 'development',
-          skipProdOnly: this.buildMode !== 'production'
+          skipProdOnly: this.buildMode !== 'production',
+          allowWrongPlatform: this.providePackageJSONForUnavailableBinaryDeps,
         }, addToGetsUsed);
       }.bind(this);
 
@@ -657,6 +668,7 @@ class Target {
           acceptableWeakPackages: this.usedPackages,
           skipDebugOnly: this.buildMode !== 'development',
           skipProdOnly: this.buildMode !== 'production',
+          allowWrongPlatform: this.providePackageJSONForUnavailableBinaryDeps,
         }, processUnibuild);
         this.unibuilds.push(unibuild);
         delete needed[unibuild.id];
@@ -792,6 +804,33 @@ class Target {
                 this.nodeModulesDirectories[unibuild.nodeModulesPath] = nmd;
               }
               f.nodeModulesDirectory = nmd;
+
+              if (!archinfo.matches(this.arch, unibuild.arch)) {
+                // The unibuild we're trying to include doesn't work for the
+                // bundle target (eg, os.osx.x86_64 instead of os.linux.x86_64)!
+                // Hopefully this is because we specially enabled the feature
+                // that leads to this.
+                if (!this.providePackageJSONForUnavailableBinaryDeps) {
+                  throw Error("mismatched arch without special feature enabled "
+                              + unibuild.pkg.name + " / " + this.arch + " / " +
+                              unibuild.arch);
+                }
+                if (!files.exists(
+                  files.pathJoin(nmd.sourcePath, '.package.json'))) {
+                  buildmessage.error(
+                    "Can't cross-compile package " +
+                      unibuild.pkg.name + ": missing .package.json");
+                  return;
+                }
+                if (!files.exists(
+                  files.pathJoin(nmd.sourcePath, '.npm-shrinkwrap.json'))) {
+                  buildmessage.error(
+                    "Can't cross-compile package " +
+                      unibuild.pkg.name + ": missing .npm-shrinkwrap.json");
+                  return;
+                }
+                nmd.writePackageJSON = true;
+              }
             }
           }
 
@@ -1197,6 +1236,8 @@ class JsImage {
 
     // Architecture required by this image
     this.arch = null;
+
+    this.providePackageJSONForUnavailableBinaryDeps = false;
   }
 
   // Load the image into the current process. It gets its own unique
@@ -1387,7 +1428,8 @@ class JsImage {
       nodeModulesDirectories.push(new NodeModulesDirectory({
         sourcePath: nmd.sourcePath,
         preferredBundlePath: modulesPhysicalLocation,
-        npmDiscards: nmd.npmDiscards
+        npmDiscards: nmd.npmDiscards,
+        writePackageJSON: nmd.writePackageJSON
       }));
     });
 
@@ -1472,13 +1514,36 @@ class JsImage {
       load.push(loadItem);
     });
 
+    const setupScriptPieces = [];
     // node_modules resources from the packages. Due to appropriate
     // builder configuration, 'meteor bundle' and 'meteor deploy' copy
     // them, and 'meteor run' symlinks them. If these contain
     // arch-specific code then the target will end up having an
     // appropriately specific arch.
     _.each(nodeModulesDirectories, function (nmd) {
-      if (nmd.sourcePath !== nmd.preferredBundlePath) {
+      if (nmd.writePackageJSON) {
+        // Make sure there's an empty node_modules directory at the right place
+        // in the tree (so that npm install puts modules there instead of
+        // elsewhere).
+        builder.reserve(
+          nmd.preferredBundlePath, {directory: true});
+        // We check that these source files exist in _emitResources when
+        // writePackageJSON is initially set.
+        builder.write(
+          files.pathJoin(files.pathDirname(nmd.preferredBundlePath),
+                         'package.json'),
+          { file: files.pathJoin(nmd.sourcePath, '.package.json') }
+        );
+        builder.write(
+          files.pathJoin(files.pathDirname(nmd.preferredBundlePath),
+                         'npm-shrinkwrap.json'),
+          { file: files.pathJoin(nmd.sourcePath, '.npm-shrinkwrap.json') }
+        );
+        // XXX does not support npmDiscards!
+
+        setupScriptPieces.push(
+          '(cd ', nmd.preferredBundlePath, ' && npm install)\n\n');
+      } else if (nmd.sourcePath !== nmd.preferredBundlePath) {
         builder.copyDirectory({
           from: nmd.sourcePath,
           to: nmd.preferredBundlePath,
@@ -1488,6 +1553,14 @@ class JsImage {
       }
     });
 
+    if (setupScriptPieces.length) {
+      setupScriptPieces.unshift('#!/bin/bash\n', 'set -e\n\n');
+      builder.write('setup.sh', {
+        data: new Buffer(setupScriptPieces.join(''), 'utf8'),
+        executable: true
+      });
+    }
+
     // Control file
     builder.writeJson('program.json', {
       format: "javascript-image-pre1",
@@ -2010,7 +2083,8 @@ exports.bundle = function ({
   includeNodeModules,
   buildOptions,
   previousBuilders,
-  hasCachedBundle
+  hasCachedBundle,
+  providePackageJSONForUnavailableBinaryDeps
 }) {
   buildOptions = buildOptions || {};
 
@@ -2077,7 +2151,8 @@ exports.bundle = function ({
         isopackCache: projectContext.isopackCache,
         arch: serverArch,
         releaseName: releaseName,
-        buildMode: buildOptions.buildMode
+        buildMode: buildOptions.buildMode,
+        providePackageJSONForUnavailableBinaryDeps
       };
       if (clientTargets)
         targetOptions.clientTargets = clientTargets;

tools/isobuild/compiler-plugin.js

@@ -518,11 +518,6 @@ _.extend(PackageSourceBatch.prototype, {
     var isopackCache = self.processor.isopackCache;
     var bundleArch = self.processor.arch;
 
-    if (! archinfo.matches(bundleArch, self.unibuild.arch))
-      throw new Error(
-        "unibuild of arch '" + self.unibuild.arch + "' does not support '" +
-          bundleArch + "'?");
-
     // Compute imports by merging the exports of all of the packages we
     // use. Note that in the case of conflicting symbols, later packages get
     // precedence.
@@ -551,6 +546,9 @@ _.extend(PackageSourceBatch.prototype, {
       // the code must access them with `Package["my-package"].MySymbol`.
       skipDebugOnly: true,
       skipProdOnly: true,
+      // We only care about getting exports here, so it's OK if we get the Mac
+      // version when we're bundling for Linux.
+      allowWrongPlatform: true,
     }, addImportsForUnibuild);
 
     // Run the linker.

tools/isobuild/compiler.js

@@ -661,6 +661,9 @@ function runLinters({inputSourceArch, isopackCache, sources,
     // the code must access them with `Package["my-package"].MySymbol`.
     skipDebugOnly: true,
     skipProdOnly: true,
+    // We only care about getting exports here, so it's OK if we get the Mac
+    // version when we're bundling for Linux.
+    allowWrongPlatform: true,
   }, (unibuild) => {
     if (unibuild.pkg.name === inputSourceArch.pkg.name)
       return;
@@ -816,6 +819,7 @@ compiler.eachUsedUnibuild = function (
   var dependencies = options.dependencies;
   var arch = options.arch;
   var isopackCache = options.isopackCache;
+  const allowWrongPlatform = options.allowWrongPlatform;
 
   var acceptableWeakPackages = options.acceptableWeakPackages || {};
 
@@ -846,7 +850,7 @@ compiler.eachUsedUnibuild = function (
     if (usedPackage.prodOnly && options.skipProdOnly)
       continue;
 
-    var unibuild = usedPackage.getUnibuildAtArch(arch);
+    var unibuild = usedPackage.getUnibuildAtArch(arch, {allowWrongPlatform});
     if (!unibuild) {
       // The package exists but there's no unibuild for us. A buildmessage has
       // already been issued. Recover by skipping.

tools/isobuild/isopack.js

@@ -438,11 +438,22 @@ _.extend(Isopack.prototype, {
   // Return the unibuild of the package to use for a given target architecture
   // (eg, 'os.linux.x86_64' or 'web'), or throw an exception if that
   // packages can't be loaded under these circumstances.
-  getUnibuildAtArch: Profile("Isopack#getUnibuildAtArch", function (arch) {
+  getUnibuildAtArch: Profile("Isopack#getUnibuildAtArch", function (
+    arch, {allowWrongPlatform} = {}) {
     var self = this;
 
-    var chosenArch = archinfo.mostSpecificMatch(
+    let chosenArch = archinfo.mostSpecificMatch(
       arch, _.pluck(self.unibuilds, 'arch'));
+    if (! chosenArch && allowWrongPlatform && arch.match(/^os\./)) {
+      // Special-case: we're looking for a specific server platform and it's
+      // not available. (eg, we're deploying from a Mac to Linux and are
+      // processing a local package with binary npm deps).  If we have "allow
+      // wrong platform" turned on, search again for the host version, which
+      // might find the Mac version.  We'll detect this case later and provide
+      // package.json instead of Mac binaries.
+      chosenArch =
+        archinfo.mostSpecificMatch(archinfo.host(), _.pluck(self.unibuilds, 'arch'));
+    }
     if (! chosenArch) {
       buildmessage.error(
         (self.name || "this app") +

tools/meteor-services/deploy.js

@@ -402,7 +402,8 @@ var bundleAndDeploy = function (options) {
     var bundleResult = bundler.bundle({
       projectContext: options.projectContext,
       outputPath: bundlePath,
-      buildOptions: options.buildOptions
+      buildOptions: options.buildOptions,
+      providePackageJSONForUnavailableBinaryDeps: !!process.env.METEOR_BINARY_DEP_WORKAROUND,
     });
 
     if (bundleResult.errors)