Automatically install missing dependencies, part 2 (#805)

src/Bundler.js

@@ -16,6 +16,7 @@ const config = require('./utils/config');
 const emoji = require('./utils/emoji');
 const loadEnv = require('./utils/env');
 const PromiseQueue = require('./utils/PromiseQueue');
+const installPackage = require('./utils/installPackage');
 const bundleReport = require('./utils/bundleReport');
 const prettifyTime = require('./utils/prettifyTime');
 
@@ -96,7 +97,8 @@ class Bundler extends EventEmitter {
       hmrHostname:
         options.hmrHostname ||
         (options.target === 'electron' ? 'localhost' : ''),
-      detailedReport: options.detailedReport || false
+      detailedReport: options.detailedReport || false,
+      autoinstall: (options.autoinstall || false) && !isProduction
     };
   }
 
@@ -325,38 +327,70 @@ class Bundler extends EventEmitter {
     }
   }
 
-  async resolveDep(asset, dep) {
+  async resolveDep(asset, dep, install = true) {
     try {
       return await this.resolveAsset(dep.name, asset.name);
     } catch (err) {
       let thrown = err;
 
       if (thrown.message.indexOf(`Cannot find module '${dep.name}'`) === 0) {
+        // Check if dependency is a local file
+        let isLocalFile = /^[/~.]/.test(dep.name);
+        let fromNodeModules = asset.name.includes(
+          `${Path.sep}node_modules${Path.sep}`
+        );
+
+        // If it's not a local file, attempt to install the dep
+        if (
+          !isLocalFile &&
+          !fromNodeModules &&
+          this.options.autoinstall &&
+          install
+        ) {
+          return await this.installDep(asset, dep);
+        }
+
+        // If the dep is optional, return before we throw
         if (dep.optional) {
           return;
         }
 
         thrown.message = `Cannot resolve dependency '${dep.name}'`;
-
-        // Add absolute path to the error message if the dependency specifies a relative path
-        if (dep.name.startsWith('.')) {
+        if (isLocalFile) {
           const absPath = Path.resolve(Path.dirname(asset.name), dep.name);
-          err.message += ` at '${absPath}'`;
-        }
-
-        // Generate a code frame where the dependency was used
-        if (dep.loc) {
-          await asset.loadIfNeeded();
-          thrown.loc = dep.loc;
-          thrown = asset.generateErrorMessage(thrown);
+          thrown.message += ` at '${absPath}'`;
         }
 
-        thrown.fileName = asset.name;
+        await this.throwDepError(asset, dep, thrown);
       }
+
       throw thrown;
     }
   }
 
+  async installDep(asset, dep) {
+    let [moduleName] = this.resolver.getModuleParts(dep.name);
+    try {
+      await installPackage([moduleName], asset.name, {saveDev: false});
+    } catch (err) {
+      await this.throwDepError(asset, dep, err);
+    }
+
+    return await this.resolveDep(asset, dep, false);
+  }
+
+  async throwDepError(asset, dep, err) {
+    // Generate a code frame where the dependency was used
+    if (dep.loc) {
+      await asset.loadIfNeeded();
+      err.loc = dep.loc;
+      err = asset.generateErrorMessage(err);
+    }
+
+    err.fileName = asset.name;
+    throw err;
+  }
+
   async processAsset(asset, isRebuild) {
     if (isRebuild) {
       asset.invalidate();

src/Logger.js

@@ -22,9 +22,24 @@ class Logger {
     this.chalk = new chalk.constructor({enabled: this.color});
   }
 
+  countLines(message) {
+    return message.split('\n').reduce((p, line) => {
+      if (process.stdout.columns) {
+        return p + Math.ceil((line.length || 1) / process.stdout.columns);
+      }
+
+      return p + 1;
+    }, 0);
+  }
+
+  writeRaw(message) {
+    this.lines += this.countLines(message) - 1;
+    process.stdout.write(message);
+  }
+
   write(message, persistent = false) {
     if (!persistent) {
-      this.lines += message.split('\n').length;
+      this.lines += this.countLines(message);
     }
 
     this._log(message);

src/cli.js

@@ -39,6 +39,7 @@ program
   .option('--no-hmr', 'disable hot module replacement')
   .option('--no-cache', 'disable the filesystem cache')
   .option('--no-source-maps', 'disable sourcemaps')
+  .option('--no-autoinstall', 'disable autoinstall')
   .option(
     '-t, --target [target]',
     'set the runtime environment, either "node", "browser" or "electron". defaults to "browser"',
@@ -74,6 +75,7 @@ program
   .option('--no-hmr', 'disable hot module replacement')
   .option('--no-cache', 'disable the filesystem cache')
   .option('--no-source-maps', 'disable sourcemaps')
+  .option('--no-autoinstall', 'disable autoinstall')
   .option(
     '-t, --target [target]',
     'set the runtime environment, either "node", "browser" or "electron". defaults to "browser"',

src/utils/PromiseQueue.js

@@ -1,9 +1,12 @@
 class PromiseQueue {
-  constructor(callback) {
+  constructor(callback, options = {}) {
     this.process = callback;
+    this.maxConcurrent = options.maxConcurrent || Infinity;
+    this.retry = options.retry !== false;
     this.queue = [];
     this.processing = new Set();
     this.processed = new Set();
+    this.numRunning = 0;
     this.runPromise = null;
     this.resolve = null;
     this.reject = null;
@@ -14,7 +17,7 @@ class PromiseQueue {
       return;
     }
 
-    if (this.runPromise) {
+    if (this.runPromise && this.numRunning < this.maxConcurrent) {
       this._runJob(job, args);
     } else {
       this.queue.push([job, args]);
@@ -41,13 +44,24 @@ class PromiseQueue {
 
   async _runJob(job, args) {
     try {
+      this.numRunning++;
       await this.process(job, ...args);
       this.processing.delete(job);
       this.processed.add(job);
+      this.numRunning--;
       this._next();
     } catch (err) {
-      this.queue.push([job, args]);
-      this.reject(err);
+      this.numRunning--;
+      if (this.retry) {
+        this.queue.push([job, args]);
+      } else {
+        this.processing.delete(job);
+      }
+
+      if (this.reject) {
+        this.reject(err);
+      }
+
       this._reset();
     }
   }
@@ -58,7 +72,7 @@ class PromiseQueue {
     }
 
     if (this.queue.length > 0) {
-      while (this.queue.length > 0) {
+      while (this.queue.length > 0 && this.numRunning < this.maxConcurrent) {
         this._runJob(...this.queue.shift());
       }
     } else if (this.processing.size === 0) {

src/utils/installPackage.js

@@ -1,54 +1,55 @@
-const spawn = require('cross-spawn');
 const config = require('./config');
-const path = require('path');
 const promisify = require('./promisify');
 const resolve = promisify(require('resolve'));
+const commandExists = require('command-exists');
+const logger = require('../Logger');
+const emoji = require('./emoji');
+const pipeSpawn = require('./pipeSpawn');
+const PromiseQueue = require('./PromiseQueue');
+const path = require('path');
+const fs = require('./fs');
 
-async function install(dir, modules, installPeers = true) {
-  let location = await config.resolve(dir, ['yarn.lock', 'package.json']);
-
-  return new Promise((resolve, reject) => {
-    let install;
-    let options = {
-      cwd: location ? path.dirname(location) : dir
-    };
-
-    if (location && path.basename(location) === 'yarn.lock') {
-      install = spawn('yarn', ['add', ...modules, '--dev'], options);
-    } else {
-      install = spawn('npm', ['install', ...modules, '--save-dev'], options);
-    }
-
-    install.stdout.pipe(process.stdout);
-    install.stderr.pipe(process.stderr);
-
-    install.on('close', async code => {
-      if (code !== 0) {
-        return reject(new Error(`Failed to install ${modules.join(', ')}.`));
-      }
-
-      if (!installPeers) {
-        return resolve();
-      }
-
-      try {
-        await Promise.all(modules.map(m => installPeerDependencies(dir, m)));
-      } catch (err) {
-        return reject(
-          new Error(
-            `Failed to install peerDependencies for ${modules.join(', ')}.`
-          )
-        );
-      }
-
-      resolve();
-    });
-  });
-}
+async function install(modules, filepath, options = {}) {
+  let {installPeers = true, saveDev = true, packageManager} = options;
+
+  logger.status(emoji.progress, `Installing ${modules.join(', ')}...`);
+
+  let packageLocation = await config.resolve(filepath, ['package.json']);
+  let cwd = packageLocation ? path.dirname(packageLocation) : process.cwd();
+
+  if (!packageManager) {
+    packageManager = await determinePackageManager(filepath);
+  }
+
+  let commandToUse = packageManager === 'npm' ? 'install' : 'add';
+  let args = [commandToUse, ...modules];
+  if (saveDev) {
+    args.push('-D');
+  } else if (packageManager === 'npm') {
+    args.push('--save');
+  }
+
+  // npm doesn't auto-create a package.json when installing,
+  // so create an empty one if needed.
+  if (packageManager === 'npm' && !packageLocation) {
+    await fs.writeFile(path.join(cwd, 'package.json'), '{}');
+  }
+
+  try {
+    await pipeSpawn(packageManager, args, {cwd});
+  } catch (err) {
+    throw new Error(`Failed to install ${modules.join(', ')}.`);
+  }
 
-async function installPeerDependencies(dir, name) {
-  let basedir = path.dirname(dir);
+  if (installPeers) {
+    await Promise.all(
+      modules.map(m => installPeerDependencies(filepath, m, options))
+    );
+  }
+}
 
+async function installPeerDependencies(filepath, name, options) {
+  let basedir = path.dirname(filepath);
   const [resolved] = await resolve(name, {basedir});
   const pkg = await config.load(resolved, ['package.json']);
   const peers = pkg.peerDependencies || {};
@@ -59,8 +60,47 @@ async function installPeerDependencies(dir, name) {
   }
 
   if (modules.length) {
-    await install(dir, modules, false);
+    await install(
+      modules,
+      filepath,
+      Object.assign({}, options, {installPeers: false})
+    );
   }
 }
 
-module.exports = install;
+async function determinePackageManager(filepath) {
+  let configFile = await config.resolve(filepath, [
+    'yarn.lock',
+    'package-lock.json'
+  ]);
+  let hasYarn = await checkForYarnCommand();
+
+  // If Yarn isn't available, or there is a package-lock.json file, use npm.
+  let configName = configFile && path.basename(configFile);
+  if (!hasYarn || configName === 'package-lock.json') {
+    return 'npm';
+  }
+
+  return 'yarn';
+}
+
+let hasYarn = null;
+async function checkForYarnCommand() {
+  if (hasYarn != null) {
+    return hasYarn;
+  }
+
+  try {
+    hasYarn = await commandExists('yarn');
+  } catch (err) {
+    hasYarn = false;
+  }
+
+  return hasYarn;
+}
+
+let queue = new PromiseQueue(install, {maxConcurrent: 1, retry: false});
+module.exports = function(...args) {
+  queue.add(...args);
+  return queue.run();
+};

src/utils/localRequire.js

@@ -13,7 +13,7 @@ async function localRequire(name, path, triedInstall = false) {
       resolved = resolve.sync(name, {basedir});
     } catch (e) {
       if (e.code === 'MODULE_NOT_FOUND' && !triedInstall) {
-        await install(path, [name]);
+        await install([name], path);
         return localRequire(name, path, true);
       }
       throw e;

src/utils/pipeSpawn.js

@@ -1,16 +1,26 @@
 const spawn = require('cross-spawn');
+const logger = require('../Logger');
 
 function pipeSpawn(cmd, params, opts) {
-  const cp = spawn(cmd, params, opts);
-  cp.stdout.pipe(process.stdout);
-  cp.stderr.pipe(process.stderr);
+  const cp = spawn(cmd, params, Object.assign({
+    env: Object.assign({
+      FORCE_COLOR: logger.color,
+      npm_config_color: logger.color ? 'always': '',
+      npm_config_progress: true
+    }, process.env)
+  }, opts));
+
+  cp.stdout.setEncoding('utf8').on('data', d => logger.writeRaw(d));
+  cp.stderr.setEncoding('utf8').on('data', d => logger.writeRaw(d));
+
   return new Promise((resolve, reject) => {
     cp.on('error', reject);
     cp.on('close', function(code) {
       if (code !== 0) {
         return reject(new Error(cmd + ' failed.'));
       }
 
+      logger.clear();
       return resolve();
     });
   });