feat(handlers): added remote tarball support (#64)

Fixes: #6

lib/finalize-manifest.js

@@ -18,7 +18,7 @@ const through = require('mississippi').through
 
 module.exports = finalizeManifest
 function finalizeManifest (pkg, spec, opts) {
-  const key = cache.key(`${spec.type}-manifest`, pkg._resolved)
+  const key = cache.key(`${spec.type}-manifest`, pkg ? pkg._resolved : spec.spec)
   opts = optCheck(opts)
   opts.memoize = true
   return (opts.cache ? cache.get.info(opts.cache, key, opts).then(res => {
@@ -30,7 +30,7 @@ function finalizeManifest (pkg, spec, opts) {
         if (err) { return cb(err) }
         // normalize should not add any fields, and once
         // makeManifest completes, it should never be modified.
-        var result = new Manifest(pkg, props)
+        var result = pkg ? new Manifest(pkg, props) : new Manifest(props)
         if (opts.cache) {
           opts.metadata = result
           cache.put(
@@ -105,17 +105,18 @@ function Manifest (pkg, fromTarball) {
 function tarballedProps (pkg, spec, opts, cb) {
   cb = dezalgo(cb)
   const extraProps = {}
-  const needsShrinkwrap = (
+  const needsShrinkwrap = (!pkg ||
     pkg._hasShrinkwrap !== false &&
     !pkg._shrinkwrap
   )
-  const needsBin = !!(
+  const needsBin = !!(!pkg ||
     !pkg.bin &&
     pkg.directories &&
     pkg.directories.bin
   )
-  const needsShasum = !pkg._shasum
-  if (!needsShrinkwrap && !needsBin && !needsShasum) {
+  const needsShasum = !pkg || !pkg._shasum
+  const needsManifest = !pkg
+  if (!needsShrinkwrap && !needsBin && !needsShasum && !needsManifest) {
     opts.log.silly('finalize-manifest', 'Skipping tarball extraction -- nothing needed.')
     return cb(null, extraProps)
   } else {
@@ -125,33 +126,43 @@ function tarballedProps (pkg, spec, opts, cb) {
     const tarData = tarball.fromManifest(pkg, spec, opts)
     let shaStream = null
     let extractorStream = null
+    let paths = []
 
-    if (needsShrinkwrap || needsBin) {
+    if (needsShrinkwrap || needsBin || needsManifest) {
       opts.log.silly('finalize-manifest', 'parsing tarball for', spec.name)
-      const dirBin = pkg.directories && pkg.directories.bin
-      extraProps.bin = pkg.bin || {}
       const dataStream = tar.extract()
       extractorStream = pipeline(gunzip(), dataStream)
       dataStream.on('entry', (header, fileStream, next) => {
         const filePath = header.name.replace(/[^/]+\//, '')
-        if (needsShrinkwrap && filePath === 'npm-shrinkwrap.json') {
-          let srData = ''
-          fileStream.on('data', d => { srData += d })
+        if (
+          (needsShrinkwrap && filePath === 'npm-shrinkwrap.json') ||
+          (needsManifest && filePath === 'package.json')
+        ) {
+          let data = ''
+          fileStream.on('data', d => { data += d })
 
           return finished(fileStream, err => {
             if (err) { return dataStream.emit('error', err) }
+            let parsed
             try {
-              extraProps._shrinkwrap = JSON.parse(srData)
+              parsed = JSON.parse(data)
               next()
             } catch (e) {
               dataStream.emit('error', e)
             }
+            if (filePath === 'package.json') {
+              Object.keys(parsed).forEach(k => {
+                if (extraProps[k] == null) {
+                  extraProps[k] = parsed[k]
+                }
+              })
+              extraProps._resolved = spec.spec
+            } else if (filePath === 'npm-shrinkwrap.json') {
+              extraProps._shrinkwrap = parsed
+            }
           })
-        } else if (needsBin && minimatch(filePath, dirBin + '/**')) {
-          const relative = path.relative(dirBin, filePath)
-          if (relative && relative[0] !== '.') {
-            extraProps.bin[path.basename(relative)] = path.join(dirBin, relative)
-          }
+        } else if (needsBin) {
+          paths.push(filePath)
         }
         // Drain and get next one
         fileStream.on('data', () => {})
@@ -173,6 +184,23 @@ function tarballedProps (pkg, spec, opts, cb) {
     // Drain the end stream
     extractorStream.on('data', () => {})
     return pipe(tarData, shaStream, extractorStream, err => {
+      if (needsBin) {
+        const dirBin = pkg
+        ? (pkg.directories && pkg.directories.bin)
+        : (extraProps.directories && extraProps.directories.bin)
+        extraProps.bin = {}
+        Object.keys((pkg && pkg.bin) || {}).forEach(k => {
+          extraProps.bin[k] = pkg.bin[k]
+        })
+        paths.forEach(filePath => {
+          if (minimatch(filePath, dirBin + '/**')) {
+            const relative = path.relative(dirBin, filePath)
+            if (relative && relative[0] !== '.') {
+              extraProps.bin[path.basename(relative)] = path.join(dirBin, relative)
+            }
+          }
+        })
+      }
       cb(err, extraProps)
     })
   }

lib/handlers/remote/manifest.js

@@ -0,0 +1,10 @@
+'use strict'
+
+module.exports = manifest
+function manifest (spec) {
+  // We can't get the manifest for a remote tarball until
+  // we extract the tarball itself.
+  // `finalize-manifest` takes care of this process of extracting
+  // a manifest based on ./tarball.js
+  return null
+}

lib/handlers/remote/tarball.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const request = require('../../registry/request')
+const url = require('url')
+
+module.exports = tarball
+function tarball (spec, opts) {
+  const uri = spec._resolved || spec.spec
+  const parsed = url.parse(uri)
+  return request.stream(uri, {
+    protocol: parsed.protocol,
+    hostname: parsed.hostname,
+    pathname: parsed.pathname
+  }, opts)
+}
+
+module.exports.fromManifest = fromManifest
+function fromManifest (manifest, spec, opts) {
+  return tarball(manifest || spec, opts)
+}

test/finalize-manifest.js

@@ -204,6 +204,46 @@ test('fills in `bin` if original was an array', t => {
   })
 })
 
+test('uses package.json as base if passed null', t => {
+  const tarballPath = 'testing/tarball-1.2.3.tgz'
+  const base = {
+    name: 'testing',
+    version: '1.2.3',
+    dependencies: { foo: '1' },
+    directories: { bin: 'foo' }
+  }
+  const sr = {
+    name: base.name,
+    version: base.version
+  }
+  return makeTarball({
+    'package.json': base,
+    'npm-shrinkwrap.json': sr,
+    'foo/x': 'x()'
+  }).then(tarData => {
+    tnock(t, OPTS.registry).get('/' + tarballPath).reply(200, tarData)
+    return finalizeManifest(null, {
+      spec: OPTS.registry + tarballPath,
+      type: 'remote'
+    }, OPTS).then(manifest => {
+      t.deepEqual(manifest, {
+        name: base.name,
+        version: base.version,
+        dependencies: base.dependencies,
+        optionalDependencies: {},
+        devDependencies: {},
+        bundleDependencies: false,
+        peerDependencies: {},
+        _resolved: OPTS.registry + tarballPath,
+        _deprecated: false,
+        _shasum: crypto.createHash('sha1').update(tarData).digest('hex'),
+        _shrinkwrap: sr,
+        bin: { 'x': path.join('foo', 'x') },
+        _id: 'testing@1.2.3'
+      }, 'entire manifest filled out from tarball')
+    })
+  })
+})
 // TODO - this is pending major changes in npm, so not implemented for now.
 test('manifest returned is immutable + inextensible')
 

test/remote.tarball.js

@@ -0,0 +1,49 @@
+'use strict'
+
+const BB = require('bluebird')
+
+const finished = BB.promisify(require('mississippi').finished)
+const mockTar = require('./util/mock-tarball')
+const npmlog = require('npmlog')
+const test = require('tap').test
+const tnock = require('./util/tnock')
+
+require('./util/test-dir')(__filename)
+
+const tarball = require('../lib/handlers/remote/tarball')
+
+npmlog.level = process.env.LOGLEVEL || 'silent'
+const OPTS = {
+  log: npmlog,
+  registry: 'https://my.mock.registry/',
+  retry: {
+    retries: 1,
+    factor: 1,
+    minTimeout: 1,
+    maxTimeout: 10
+  }
+}
+
+test('basic tarball streaming', function (t) {
+  const pkg = {
+    'package.json': JSON.stringify({
+      name: 'foo',
+      version: '1.2.3'
+    }),
+    'index.js': 'console.log("hello world!")'
+  }
+  return mockTar(pkg).then(tarData => {
+    const tarballPath = '/foo/hosted/plexus/foo-1.2.3.tgz'
+    const srv = tnock(t, OPTS.registry)
+    srv.get(tarballPath).reply(200, tarData)
+    let data = ''
+    return finished(
+      tarball({
+        type: 'remote',
+        spec: OPTS.registry + tarballPath.slice(1)
+      }, OPTS).on('data', d => { data += d })
+    ).then(() => {
+      t.equal(data, tarData, 'fetched tarball data matches one from server')
+    })
+  })
+})