feat: support high speed store for tarball download (#276)

.gitignore

@@ -26,3 +26,4 @@ test/fixtures/css-loader-example2/node_modules
 test/fixtures/css-loader-example2/webpack*.json
 !test/fixtures/warn-node-modules-exists/node_modules
 !test/fixtures/ignore-warn-node-modules-exists/node_modules
+test/fixtures/high-speed-store/tmp

bin/install.js

@@ -27,6 +27,8 @@ const argv = parseArgs(orignalArgv, {
     // {"http://a.com":"http://b.com"}
     'tarball-url-mapping',
     'proxy',
+    // --high-speed-store=filepath
+    'high-speed-store',
   ],
   boolean: [
     'version',
@@ -106,6 +108,7 @@ Options:
   --cache-strict: use disk cache even on production env.
   --fix-bug-versions: auto fix bug version of package.
   --prune: prune unnecessary files from ./node_modules, such as markdown, typescript source files, and so on.
+  --high-speed-store: specify high speed store script to cache tgz files, and so on. Should export '* getStream(url)' function.
 `
   );
   process.exit(0);
@@ -248,6 +251,10 @@ co(function* () {
     };
   }
 
+  if (argv['high-speed-store']) {
+    config.highSpeedStore = require(argv['high-speed-store']);
+  }
+
   // -g install to npm's global prefix
   if (argv.global) {
     // support custom prefix for global install

lib/download/npm.js

@@ -308,6 +308,23 @@ function* getTarballStream(tarballUrl, pkg, options) {
     };
   }
 
+  // high speed store
+  if (options.highSpeedStore) {
+    try {
+      const stream = yield options.highSpeedStore.getStream(tarballUrl);
+      if (stream) {
+        // record size
+        stream.on('data', chunk => {
+          options.totalTarballSize += chunk.length;
+        });
+        return stream;
+      }
+    } catch (err) {
+      options.console.warn('[npminstall:download:npm] highSpeedStore.get %s error: %s', tarballUrl, err);
+      options.console.warn(err.stack);
+    }
+  }
+
   if (!options.cacheDir || utils.isSudo()) {
     // sudo don't touch the cacheDir
     // production mode
@@ -344,11 +361,13 @@ function* getTarballStream(tarballUrl, pkg, options) {
   let exists = yield fs.exists(tarballFile);
   if (!exists) {
     // try to remove expired tmp dirs
-    // last year
-    const lastYearTmpDir = path.join(options.cacheDir, '.tmp', moment().subtract(1, 'years').format('YYYY'));
-    // last month
-    const lastMonthTmpDir = path.join(options.cacheDir, '.tmp', moment().subtract(1, 'months').format('YYYY/MM'));
-    const dirs = [ lastYearTmpDir, lastMonthTmpDir ];
+    const dirs = [];
+    for (let i = 1; i <= 3; i++) {
+      // year
+      dirs.push(path.join(options.cacheDir, '.tmp', moment().subtract(i, 'years').format('YYYY')));
+      // month
+      dirs.push(path.join(options.cacheDir, '.tmp', moment().subtract(i, 'months').format('YYYY/MM')));
+    }
     for (const dir of dirs) {
       try {
         yield utils.rimraf(dir);

lib/local_install.js

@@ -136,6 +136,7 @@ function* _install(options) {
   }
 
   yield parallel(tasks, 10);
+  options.downloadFinished = Date.now();
   options.spinner && options.spinner.succeed(`Installed ${tasks.length} packages`);
 
   // link every packages' latest version to target directory
@@ -469,15 +470,17 @@ function recordPackageVersions(options) {
 
 function finishInstall(options) {
   const totalUse = Date.now() - options.start;
+  const downloadUse = options.downloadFinished - options.start;
   const totalSize = options.totalTarballSize + options.totalJSONSize;
-  const avgSpeed = totalSize / totalUse * 1000;
+  const avgSpeed = totalSize / downloadUse * 1000;
   const logArguments = [
-    chalk[options.detail ? 'green' : 'white']('All packages installed (%s%s%s%sused %s, speed %s/s, json %s(%s), tarball %s%s)'),
+    chalk[options.detail ? 'green' : 'white']('All packages installed (%s%s%s%sused %s(network %s), speed %s/s, json %s(%s), tarball %s%s)'),
     options.registryPackages ? `${options.registryPackages} packages installed from npm registry, ` : '',
     options.remotePackages ? `${options.remotePackages} packages installed from remote url, ` : '',
     options.localPackages ? `${options.localPackages} packages installed from local file, ` : '',
     options.gitPackages ? `${options.gitPackages} packages installed from git, ` : '',
     ms(totalUse),
+    ms(downloadUse),
     bytes(avgSpeed),
     options.totalJSONCount,
     bytes(options.totalJSONSize),

package.json

@@ -38,7 +38,8 @@
     "mkdirp": "^0.5.1",
     "moment": "^2.18.1",
     "ms": "^2.0.0",
-    "mz": "^2.6.0",
+    "mz": "^2.7.0",
+    "mz-modules": "^2.1.0",
     "node-gyp": "^3.7.0",
     "node-homedir": "^1.0.0",
     "normalize-git-url": "^3.0.2",
@@ -50,7 +51,7 @@
     "semver": "^5.3.0",
     "tar": "^4.0.1",
     "urllib": "^2.24.0",
-    "utility": "^1.12.0",
+    "utility": "^1.14.0",
     "uuid": "^3.0.1"
   },
   "devDependencies": {

test/all.js

@@ -4,11 +4,10 @@ const co = require('co');
 const path = require('path');
 const rimraf = require('rimraf');
 const semver = require('semver');
-const npminstall = require('./npminstall');
 const spawn = require('child_process').spawn;
+const npminstall = require('./npminstall');
 
 const names = [
-  'strongloop',
   'express', 'koa', 'browserify',
   'pm2',
   'grunt-cli',
@@ -32,6 +31,9 @@ const semvers = {
   '>= 6': [
     'egg',
   ],
+  '<= 8': [
+    'strongloop',
+  ],
 };
 
 for (const version in semvers) {

test/fixtures/high-speed-store/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "high-speed-store",
+  "devDependencies": {
+    "@types/react": "15.0.4",
+    "@types/react-dom": "0.14.18"
+  }
+}

test/fixtures/high-speed-store/store.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const co = require('co');
+const path = require('path');
+const fs = require('mz/fs');
+const mkdirp = require('mz-modules/mkdirp');
+const utility = require('utility');
+const urllib = require('urllib');
+
+exports.getStream = function* (url) {
+  const dir = path.join(__dirname, 'tmp');
+  yield mkdirp(dir);
+  const file = path.join(dir, utility.md5(url) + path.extname(url));
+  if (yield fs.exists(file)) {
+    return fs.createReadStream(file);
+  }
+
+  const writeStream = fs.createWriteStream(file);
+  yield urllib.request(url, {
+    writeStream,
+    timeout: 10000,
+    followRedirect: true,
+  });
+  return fs.createReadStream(file);
+};

test/high-speed-store.test.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const coffee = require('coffee');
+const rimraf = require('rimraf');
+const path = require('path');
+const assert = require('assert');
+const fs = require('fs');
+
+describe('test/high-speed-store.test.js', () => {
+  const cwd = path.join(__dirname, 'fixtures', 'high-speed-store');
+  const storeScript = path.join(cwd, 'store.js');
+  const bin = path.join(__dirname, '../bin/install.js');
+
+  function cleanup() {
+    rimraf.sync(path.join(cwd, 'node_modules'));
+    rimraf.sync(path.join(cwd, 'tmp'));
+  }
+
+  beforeEach(cleanup);
+  afterEach(cleanup);
+
+  it('should get tarball stream from store', function* () {
+    yield coffee.fork(bin, [ '-d', `--high-speed-store=${storeScript}` ], { cwd })
+      .debug()
+      .expect('code', 0)
+      .expect('stdout', /All packages installed/)
+      .end();
+    const pkg = require(path.join(cwd, 'node_modules/@types/react-dom/node_modules/@types/react/package.json'));
+    assert(pkg.version === '15.0.4');
+    assert(fs.readdirSync(path.join(cwd, 'tmp')).length === 2);
+  });
+});

test/tarball-url-mapping.test.js

@@ -5,7 +5,7 @@ const rimraf = require('rimraf');
 const mkdirp = require('mkdirp');
 const path = require('path');
 
-describe('tarball-url-mapping.test.js', () => {
+describe('test/tarball-url-mapping.test.js', () => {
   const tmp = path.join(__dirname, 'fixtures', 'tmp');
   const bin = path.join(__dirname, '../bin/install.js');
 

test/use-exists-version.test.js

@@ -5,7 +5,7 @@ const rimraf = require('rimraf');
 const path = require('path');
 const assert = require('assert');
 
-describe('use-exists-version.test.js', () => {
+describe('test/use-exists-version.test.js', () => {
   const tmp = path.join(__dirname, 'fixtures', 'try-to-use-one-version');
   const bin = path.join(__dirname, '../bin/install.js');