Merge 53fc6a6 into 1a3e8e9

extract.js

@@ -1,7 +1,7 @@
 'use strict'
 
 var cache = require('./lib/cache')
-var extractStream = require('./lib/util/extract-stream')
+var extractStream = require('./lib/extract-stream')
 var pipe = require('mississippi').pipe
 var optCheck = require('./lib/util/opt-check')
 var rps = require('realize-package-specifier')

lib/util/extract-stream.js → lib/extract-stream.js

@@ -1,6 +1,6 @@
 'use strict'
 
-var gunzip = require('./gunzip-maybe')
+var gunzip = require('./util/gunzip-maybe')
 var path = require('path')
 var pipeline = require('mississippi').pipeline
 var tar = require('tar-fs')

lib/finalize-manifest.js

@@ -0,0 +1,129 @@
+'use strict'
+
+var checksumStream = require('./util/checksum-stream')
+var dezalgo = require('dezalgo')
+var finished = require('mississippi').finished
+var gunzip = require('./util/gunzip-maybe')
+var minimatch = require('minimatch')
+var normalize = require('normalize-package-data')
+var optCheck = require('./util/opt-check')
+var path = require('path')
+var pipe = require('mississippi').pipe
+var pipeline = require('mississippi').pipeline
+var tar = require('tar-stream')
+var through = require('mississippi').through
+
+module.exports = finalizeManifest
+function finalizeManifest (pkg, spec, where, opts, cb) {
+  completeFromTarball(pkg, spec, where, opts, function (err) {
+    if (err) { return cb(err) }
+    // normalize should not add any fields, and once
+    // makeManifest completes, it should never be modified.
+    cb(null, new Manifest(pkg))
+  })
+}
+
+module.exports.Manifest = Manifest
+function Manifest (pkg) {
+  this.name = pkg.name
+  this.version = pkg.version
+  this.dependencies = pkg.dependencies || {}
+  this.optionalDependencies = pkg.optionalDependencies || {}
+  this.devDependencies = pkg.devDependencies || {}
+  var bundled = (
+    pkg.bundledDependencies ||
+    pkg.bundleDependencies ||
+    false
+  )
+  this.bundleDependencies = !!bundled.length || false
+  this.peerDependencies = pkg.peerDependencies || {}
+
+  // This one depends entirely on each handler.
+  this._resolved = pkg._resolved
+
+  // Filled in by completeFromTarball as needed.
+  this._shasum = pkg._shasum
+  this._shrinkwrap = pkg._shrinkwrap || null
+  this.bin = pkg.bin || null
+
+  this._id = null // filled in by normalize-package-data, but unnecessary
+
+  Object.preventExtensions(this)
+  normalize(this)
+  Object.freeze(this)
+}
+
+// Some things aren't filled in by standard manifest fetching.
+// If this function needs to do its work, it will grab the
+// package tarball, extract it, and take whatever it needs
+// from the stream.
+function completeFromTarball (pkg, spec, where, opts, cb) {
+  cb = dezalgo(cb)
+  var needsShrinkwrap = !(
+    pkg._hasShrinkwrap === false ||
+    pkg._shrinkwrap
+  )
+  var needsBin = !!(
+    !pkg.bin &&
+    pkg.directories &&
+    pkg.directories.bin
+  )
+  var needsShasum = !pkg._shasum
+  if (!needsShrinkwrap && !needsBin && !needsShasum) {
+    opts.log.silly('finalize-manifest', 'Skipping tarball extraction -- nothing needed.')
+    return cb(null)
+  } else {
+    opts = optCheck(opts)
+    opts.memoize = false
+    var tarball = require('./handlers/' + spec.type + '/tarball')
+    var tarData = tarball.fromManifest(pkg, spec, opts)
+    var shaStream = null
+    var extractorStream = null
+
+    if (needsShrinkwrap || needsBin) {
+      opts.log.silly('finalize-manifest', 'parsing tarball for', spec.name)
+      var dirBin = pkg.directories && pkg.directories.bin
+      pkg.bin = pkg.bin || {}
+      var dataStream = tar.extract()
+      extractorStream = pipeline(gunzip(), dataStream)
+      dataStream.on('entry', function doEntry (header, fileStream, next) {
+        var filePath = header.name.replace(/[^/]+\//, '')
+        if (needsShrinkwrap && filePath === 'npm-shrinkwrap.json') {
+          var srData = ''
+          fileStream.on('data', function (d) { srData += d })
+
+          return finished(fileStream, function (err) {
+            if (err) { return dataStream.emit('error', err) }
+            try {
+              pkg._shrinkwrap = JSON.parse(srData)
+              next()
+            } catch (e) {
+              dataStream.emit('error', e)
+            }
+          })
+        } else if (needsBin && minimatch(filePath, dirBin + '/**')) {
+          var relative = path.relative(dirBin, filePath)
+          if (relative && relative[0] !== '.') {
+            pkg.bin[path.basename(relative)] = path.join(dirBin, relative)
+          }
+        }
+        // Drain and get next one
+        fileStream.on('data', function () {})
+        next()
+      })
+    } else {
+      extractorStream = through()
+    }
+    if (needsShasum) {
+      shaStream = checksumStream(null, opts.hashAlgorithm)
+      shaStream.on('digest', function (d) {
+        pkg._shasum = d
+      })
+    } else {
+      shaStream = through()
+    }
+    // Drain the end stream
+    extractorStream.on('data', function () {})
+    return pipe(tarData, shaStream, extractorStream, cb)
+  }
+}

lib/registry/manifest.js

@@ -1,12 +1,10 @@
 'use strict'
 
-var extractShrinkwrap = require('../util/extract-shrinkwrap')
 var optCheck = require('../util/opt-check')
 var pickManifest = require('./pick-manifest')
 var pickRegistry = require('./pick-registry')
 var url = require('url')
 var request = require('./request')
-var tarball = require('./tarball')
 
 module.exports = manifest
 function manifest (spec, opts, cb) {
@@ -28,13 +26,21 @@ function manifest (spec, opts, cb) {
       defaultTag: opts.defaultTag
     }, function (err, manifest) {
       if (err) { return cb(err) }
-      opts.log.silly(
-        'registry.manifest',
-        spec.name + '@' + spec.spec,
-        'resolved to',
-        manifest.name + '@' + manifest.version
+      // Done here instead of ./finalize-manifest because these fields
+      // have to be filled in differently depending on type.
+      manifest._shasum = (
+        manifest.dist && manifest.dist.shasum
+      ) || manifest._shasum
+      manifest._resolved = (
+        manifest.dist && manifest.dist.tarball
+      ) || url.resolve(
+        registry,
+        '/' + manifest.name +
+        '/-/' + manifest.name +
+        '-' + manifest.version +
+        '.tgz'
       )
-      ensureShrinkwrap(manifest, registry, opts, cb)
+      cb(null, manifest)
     })
   })
 }
@@ -45,31 +51,3 @@ function metadataUrl (registry, name) {
   : registry
   return url.resolve(normalized, name)
 }
-
-function ensureShrinkwrap (manifest, registry, opts, cb) {
-  if (manifest._hasShrinkwrap === false || manifest._shrinkwrap || !manifest.dist) {
-    opts.log.silly('registry.manifest', 'shrinkwrap extraction not needed')
-    cb(null, manifest)
-  } else {
-    opts.log.silly('registry.manifest', 'extracting shrinkwrap')
-    opts.memoize = false
-    var shrinkwrap
-    var tarstream = tarball.fromManifest(manifest, registry, opts)
-    extractShrinkwrap(tarstream, opts, function (err, sr) {
-      if (err) { return cb(err) }
-      shrinkwrap = sr
-    })
-    tarstream.on('data', function () {})
-    tarstream.on('error', cb)
-    tarstream.on('end', function () {
-      manifest._hasShrinkwrap = !!shrinkwrap
-      manifest._shrinkwrap = shrinkwrap
-      if (shrinkwrap) {
-        opts.log.silly('registry.manifest', 'shrinkwrap found')
-      } else {
-        opts.log.silly('registry.manifest', 'no shrinkwrap')
-      }
-      cb(null, manifest)
-    })
-  }
-}

lib/registry/tarball.js

@@ -10,7 +10,6 @@ var through = require('mississippi').through
 module.exports = tarball
 function tarball (spec, opts) {
   opts = optCheck(opts)
-  var registry = pickRegistry(spec, opts)
   opts.log.verbose(
     'registry.tarball',
     'looking up registry-based metadata for ', spec
@@ -23,17 +22,18 @@ function tarball (spec, opts) {
       'registry metadata found. Downloading ', manifest.name + '@' + manifest.version
     )
     pipe(
-      fromManifest(manifest, registry),
+      fromManifest(manifest, spec, opts),
       stream
     )
   })
   return stream
 }
 
 module.exports.fromManifest = fromManifest
-function fromManifest (manifest, registry, opts) {
+function fromManifest (manifest, spec, opts) {
   opts = optCheck(opts)
-  var uri = manifest.dist && manifest.dist.tarball
-  opts.digest = manifest.dist && manifest.dist.shasum
+  var registry = pickRegistry(spec, opts)
+  var uri = manifest._resolved
+  opts.digest = manifest._shasum
   return request.stream(uri, registry, opts)
 }

lib/util/checksum-stream.js

@@ -0,0 +1,26 @@
+'use strict'
+
+var crypto = require('crypto')
+var through = require('mississippi').through
+
+module.exports = checksumStream
+function checksumStream (digest, algorithm) {
+  var hash = crypto.createHash(algorithm || 'sha1')
+  var stream = through(function (chunk, enc, cb) {
+    hash.update(chunk, enc)
+    cb(null, chunk, enc)
+  }, function (cb) {
+    var streamDigest = hash.digest('hex')
+    if (digest && streamDigest !== digest) {
+      var err = new Error('checksum failed')
+      err.code = 'EBADCHECKSUM'
+      err.expected = digest
+      err.found = streamDigest
+      return cb(err)
+    } else {
+      stream.emit('digest', streamDigest)
+      cb()
+    }
+  })
+  return stream
+}

manifest.js

@@ -1,5 +1,6 @@
 'use strict'
 
+var finalizeManifest = require('./lib/finalize-manifest')
 var optCheck = require('./lib/util/opt-check')
 var rps = require('realize-package-specifier')
 
@@ -17,7 +18,8 @@ function manifest (spec, opts, cb) {
     if (err) { return cb(err) }
     var fetcher = handlers[res.type] || (handlers[res.type] = require('./lib/handlers/' + res.type + '/manifest'))
     fetcher(res, opts, function (err, mani) {
-      cb(err, mani)
+      if (err) { return cb(err) }
+      finalizeManifest(mani, res, opts.where, opts, cb)
     })
   })
 }

package.json

@@ -25,6 +25,7 @@
     "cacache": "^5.0.0",
     "dezalgo": "^1.0.3",
     "inflight": "^1.0.6",
+    "minimatch": "^3.0.3",
     "mississippi": "^1.2.0",
     "npm-registry-client": "^7.4.1",
     "realize-package-specifier": "^3.0.3",

test/extract-stream.js

@@ -0,0 +1,23 @@
+'use strict'
+
+var fs = require('fs')
+var npmlog = require('npmlog')
+var pipe = require('mississippi').pipe
+var test = require('tap').test
+
+require('./util/test-dir')(__filename)
+
+var extractStream = require('../lib/extract-stream')
+
+npmlog.level = process.env.LOGLEVEL || 'silent'
+var OPTS = {
+  log: npmlog
+}
+
+test('basic extraction')
+test('excludes symlinks')
+test('renames .gitignore to .npmignore if not present')
+test('accepts gid and uid opts')
+test('accepts dmode/fmode/umask opts')
+test('automatically handles gzipped tarballs')
+test('strips first item in path, even if not `package/`')