Merge pull request #804 from spmjs/install-with-stream

improve spm install
spmjs · May 30, 2014 · ee63b4f · ee63b4f
2 parents 523cffd + 479e6e5
commit ee63b4f
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 131 deletions.
diff --git a/lib/install.js b/lib/install.js
@@ -7,12 +7,16 @@ var color = require('colorful');
 var _ = require('lodash');
 var spmrc = require('spmrc');
 var md5file = require('./utils').md5file;
-var tar = require('./utils/tar');
 var log = require('./utils/log');
 var yuan = require('./sdk/yuan');
 var iduri = require('./sdk/iduri');
 var file = require('./sdk/file');
 var mo = require('./sdk/module');
+var gulp = require('gulp');
+var gunzip = require('gulp-gunzip');
+var untar = require('gulp-untar2');
+var source = require('vinyl-source-stream');
+var pipe = require('multipipe');
 
 var downloaded = {};
 
@@ -128,32 +132,31 @@ function spmInstall(data, callback, saveDeps) {
       log.info('install', color.magenta(pkgId));
       callback(null, pkgId);
     },
-    fetch,
-    copy
-  ], function(err, pkg) {
+    fetch
+  ], function(err, dest) {
     if (err) {
       callback(err);
       return;
     }
 
+    var relativePath = path.relative(process.cwd(), dest);
+    log.info('installed', color.green(relativePath));
+    var pkg = file.readJSON(path.join(dest, 'package.json'));
+
     // save dependencies to package.json
     if (saveDeps) {
       save(pkg.name, pkg.version);
     }
 
     var src = pkg.src;
-    var packages = parseDependencies(path.join(src, 'package.json'));
+    var packages = parseDependencies(pkg);
     if (packages.length) {
       log.info('depends', packages.join(', '));
     }
 
     var id = pkg.name + '@' + pkg.version;
     downloaded[id] = pkg;
 
-    if (file.exists(src)) {
-      file.rmdir(src, {force: true});
-    }
-
     if (packages.length) {
       queueInstall(packages, function(err) {
         callback(err, pkg);
@@ -187,9 +190,15 @@ function fetch(query, callback) {
       filename = body.name + '-' + body.version + '.tar.gz';
     }
 
-    var dest = path.join(config.cache, filename);
-    if (!config.force && file.exists(dest) && md5file(dest) === body.md5) {
-      extract(dest, callback);
+    var dest = path.join(config.dest, body.name, body.version);
+    var cacheDest = path.join(config.cache, filename);
+
+    if (!config.force && file.exists(dest)) {
+      var pkgId = body.name + '@' + body.version;
+      log.info('found', pkgId);
+      callback(null, dest);
+    } else if (!config.force && file.exists(cacheDest) && md5file(cacheDest) === body.md5) {
+      extract(cacheDest, dest, callback);
     } else {
       fetchTarball('repository/' + body.name + '/' + body.version + '/' + filename, dest, callback);
     }
@@ -201,61 +210,35 @@ function fetchTarball(urlpath, dest, callback) {
   log.info('download', urlpath);
 
   var data = {urlpath: urlpath, method: 'GET', encoding: null};
-  yuan(config).request(data, function(err, res, body) {
-    fs.writeFile(dest, body, function(err) {
-      if (err) {
-        callback(err);
-      } else {
-        log.info('save', dest);
-        extract(dest, callback);
-      }
-    });
-  });
-}
 
-function copy(src, callback) {
-  var pkg = file.readJSON(path.join(src, 'package.json'));
-  var dest;
-
-  // install to name/version/
-  if (pkg.name && pkg.version) {
-    dest = path.join(config.dest, pkg.name, pkg.version);
-  } else {
-    if (!pkg.name) {
-      log.error('package', 'missing `name` in package.json');
-    }
-    if (!pkg.version) {
-      log.error('package', 'missing `version` in package.json');
-    }
-    process.exit(1);
-  }
-  log.info('installed', color.green(dest));
-
-  // copy package.json
-  file.copy(
-    path.join(src, 'package.json'), path.join(dest, 'package.json')
+  var stream = pipe(
+    yuan(config).request(data),
+    source(path.basename(urlpath)),
+    gulp.dest(config.cache),
+    gunzip(),
+    untar(),
+    gulp.dest(dest)
   );
-
-  // fix windows path
-  file.recurse(src, function(fpath) {
-    var fname = path.relative(src, fpath);
-    file.copy(fpath, path.join(dest, fname));
+  stream.on('error', callback);
+  stream.on('end', function() {
+    callback(null, dest);
   });
-
-  pkg.src = src;
-  callback(null, pkg);
+  stream.resume();
 }
 
-function extract(src, callback) {
-  var tmp = spmrc.get('user.temp');
-  var random = parseInt(Math.random() * 1000000000, 10).toString();
-  tmp = path.join(tmp, random, path.basename(src)).replace(/\.tar\.gz$/, '');
-  if (file.exists(tmp)) {
-    file.rmdir(tmp, {force: true});
-  }
+function extract(src, dest, callback) {
   log.info('extract', src);
-  log.debug('extract', tmp);
-  tar.extract(src, tmp, callback);
+  var stream = pipe(
+    gulp.src(src),
+    gunzip(),
+    untar(),
+    gulp.dest(dest)
+  );
+  stream.on('error', callback);
+  stream.on('end', function() {
+    callback(null, dest);
+  });
+  stream.resume();
 }
 
 function parseDependencies(pkg, includeDev) {

diff --git a/lib/sdk/yuan.js b/lib/sdk/yuan.js
@@ -41,6 +41,8 @@ function Yuan(options) {
 }
 
 Yuan.prototype.request = function(data, callback) {
+  callback = callback || function() {};
+
   data.url = util.format('%s/%s', this.server, data.urlpath);
   if (data.method.toLowerCase() === 'get' && data.urlpath.indexOf('repository/search') !== 0) {
     // mirror is faster

diff --git a/lib/utils/tar.js b/lib/utils/tar.js
@@ -74,75 +74,6 @@ exports.create = function(source, target, callback) {
   istream.pipe(packer).pipe(zipper).pipe(fwriter);
 };
 
-exports.extract = function(source, target, callback) {
-  var umask = modes.umask;
-  var dmode = modes.exec;
-  var fmode = modes.file;
-
-  function returnError(err) {
-    // don't call the callback multiple times, just return the first error
-    var _callback = callback;
-    callback = function() {};
-    return _callback(err);
-  }
-
-  var freader = source instanceof Stream ? source : fs.createReadStream(source);
-  freader.on('error', function(err) {
-    log.error('reading', source);
-    return returnError(err);
-  });
-
-  var extract_opts = {
-    type: 'Directory',
-    path: target,
-    // strip: 0,
-    filter: function() {
-      // symbolic links are not allowed in packages
-      if (this.type.match(/^.*Link$/)) {
-        log.warn('excluding',
-            this.path.substr(target.length + 1) + ' -> ' + this.linkpath
-        );
-        return false;
-      }
-      return true;
-    }
-  };
-  if (!isWindows && typeof myUid === 'number' && typeof myGid === 'number') {
-    extract_opts.uid = myUid;
-    extract_opts.gid = myGid;
-  }
-
-  var extractor = tar.Extract(extract_opts);
-  extractor.on('error', function(err) {
-    log.error('untar', source);
-    return returnError(err);
-  });
-  extractor.on('entry', function(entry) {
-    entry.mode = entry.mode || entry.props.mode;
-    entry.mode = entry.mode | (entry.type === 'Directory' ? dmode : fmode);
-    entry.mode = entry.mode & (~umask);
-    entry.props.mode = entry.mode;
-
-    if (!isWindows && typeof myUid === 'number' && typeof myGid === 'number') {
-      entry.props.uid = entry.uid = myUid;
-      entry.props.gid = entry.gid = myGid;
-    }
-  });
-
-  extractor.on('end', function() {
-    return callback(null, target);
-  });
-
-  var unzipper = zlib.Unzip();
-  unzipper.on('error', function(err) {
-    log.error('unzip', source);
-    return returnError(err);
-  });
-
-  freader.pipe(unzipper).pipe(extractor);
-};
-
-
 
 var Ignore = require('fstream-ignore');
 var inherits = require('inherits');

diff --git a/package.json b/package.json
@@ -49,7 +49,10 @@
     "umi": "~0.2.0",
     "gulp": "~3.6.2",
     "gulp-if": "~1.1.0",
-    "multipipe": "~0.0.2"
+    "multipipe": "~0.1.0",
+    "gulp-gunzip": "0.0.2",
+    "gulp-untar2": "0.0.3",
+    "vinyl-source-stream": "~0.1.0"
   },
   "devDependencies": {
     "sinon": "*",