Skip to content
This repository has been archived by the owner on Jul 3, 2019. It is now read-only.

Commit

Permalink
Merge b8ff004 into 492db37
Browse files Browse the repository at this point in the history
  • Loading branch information
zkat committed Feb 18, 2017
2 parents 492db37 + b8ff004 commit e51a269
Show file tree
Hide file tree
Showing 16 changed files with 311 additions and 231 deletions.
47 changes: 30 additions & 17 deletions extract.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict'

var cache = require('./lib/cache')
var extractStream = require('./lib/util/extract-stream')
var extractStream = require('./lib/extract-stream')
var pipe = require('mississippi').pipe
var optCheck = require('./lib/util/opt-check')
var rps = require('realize-package-specifier')
Expand All @@ -13,23 +13,36 @@ function extract (spec, dest, opts, cb) {
opts = null
}
opts = optCheck(opts)
var xtractor = extractStream(dest, opts)
var caStream = cache.get.stream.byDigest(opts.cache, opts.digest, opts)
if (opts.digest) {
opts.log.silly('extract', 'trying from digest:', opts.digest)
opts.log.silly('extract', 'trying ', spec, ' digest:', opts.digest)
extractByDigest(dest, opts, function (err) {
if (err && err === 'ENOENT') {
opts.log.silly('extract', 'digest for', spec, 'not present. Using manifest.')
return extractByManifest(spec, dest, opts, cb)
} else {
return cb(err)
}
})
} else {
opts.log.silly('extract', 'no digest provided for ', spec, '- extracting by manifest')
extractByManifest(spec, dest, opts, cb)
}
caStream.on('error', function (err) {
if (err && err.code !== 'ENOENT') { return cb(err) }
rps(spec, function (err, res) {
if (err) { return cb(err) }
var tarball = require('./lib/handlers/' + res.type + '/tarball')
pipe(tarball(res, opts), xtractor, function (err) {
opts.log.silly('extract', 'extraction finished for', spec)
cb(err)
})
}

function extractByDigest (dest, opts, cb) {
var xtractor = extractStream(dest, opts)
var cached = cache.get.stream.byDigest(opts.cache, opts.digest, opts)
pipe(cached, xtractor, cb)
}

function extractByManifest (spec, dest, opts, cb) {
var xtractor = extractStream(dest, opts)
rps(spec, function (err, res) {
if (err) { return cb(err) }
var tarball = require('./lib/handlers/' + res.type + '/tarball')
pipe(tarball(res, opts), xtractor, function (err) {
opts.log.silly('extract', 'extraction finished for', spec)
cb(err)
})
}).on('end', function () {
opts.log.silly('extract', spec, 'extracted by digest')
cb(null)
}).pipe(xtractor)
})
}
2 changes: 1 addition & 1 deletion lib/util/extract-stream.js → lib/extract-stream.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

var gunzip = require('./gunzip-maybe')
var gunzip = require('./util/gunzip-maybe')
var path = require('path')
var pipeline = require('mississippi').pipeline
var tar = require('tar-fs')
Expand Down
133 changes: 133 additions & 0 deletions lib/finalize-manifest.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
'use strict'

var checksumStream = require('./util/checksum-stream')
var dezalgo = require('dezalgo')
var finished = require('mississippi').finished
var gunzip = require('./util/gunzip-maybe')
var minimatch = require('minimatch')
var normalize = require('normalize-package-data')
var optCheck = require('./util/opt-check')
var path = require('path')
var pipe = require('mississippi').pipe
var pipeline = require('mississippi').pipeline
var tar = require('tar-stream')
var through = require('mississippi').through

module.exports = finalizeManifest
function finalizeManifest (pkg, spec, opts, cb) {
tarballedProps(pkg, spec, opts, function (err, props) {
if (err) { return cb(err) }
// normalize should not add any fields, and once
// makeManifest completes, it should never be modified.
cb(null, new Manifest(pkg, props))
})
}

module.exports.Manifest = Manifest
function Manifest (pkg, fromTarball) {
fromTarball = fromTarball || {}
this.name = pkg.name
this.version = pkg.version
this.dependencies = pkg.dependencies || {}
this.optionalDependencies = pkg.optionalDependencies || {}
this.devDependencies = pkg.devDependencies || {}
var bundled = (
pkg.bundledDependencies ||
pkg.bundleDependencies ||
false
)
this.bundleDependencies = bundled
this.peerDependencies = pkg.peerDependencies || {}

// This one depends entirely on each handler.
this._resolved = pkg._resolved

// We sometimes need to grab these from the tarball.
this._shasum = pkg._shasum || fromTarball._shasum
this._shrinkwrap = pkg._shrinkwrap || fromTarball._shrinkwrap || null
this.bin = pkg.bin || fromTarball.bin || null

this._id = null // filled in by normalize-package-data, but unnecessary

Object.preventExtensions(this)
normalize(this)
Object.freeze(this)
}

// Some things aren't filled in by standard manifest fetching.
// If this function needs to do its work, it will grab the
// package tarball, extract it, and take whatever it needs
// from the stream.
function tarballedProps (pkg, spec, opts, cb) {
cb = dezalgo(cb)
var extraProps = {}
var needsShrinkwrap = (
pkg._hasShrinkwrap !== false &&
!pkg._shrinkwrap
)
var needsBin = !!(
!pkg.bin &&
pkg.directories &&
pkg.directories.bin
)
var needsShasum = !pkg._shasum
if (!needsShrinkwrap && !needsBin && !needsShasum) {
opts.log.silly('finalize-manifest', 'Skipping tarball extraction -- nothing needed.')
return cb(null, extraProps)
} else {
opts = optCheck(opts)
opts.memoize = false
var tarball = require('./handlers/' + spec.type + '/tarball')
var tarData = tarball.fromManifest(pkg, spec, opts)
var shaStream = null
var extractorStream = null

if (needsShrinkwrap || needsBin) {
opts.log.silly('finalize-manifest', 'parsing tarball for', spec.name)
var dirBin = pkg.directories && pkg.directories.bin
pkg.bin = pkg.bin || {}
var dataStream = tar.extract()
extractorStream = pipeline(gunzip(), dataStream)
dataStream.on('entry', function doEntry (header, fileStream, next) {
var filePath = header.name.replace(/[^/]+\//, '')
if (needsShrinkwrap && filePath === 'npm-shrinkwrap.json') {
var srData = ''
fileStream.on('data', function (d) { srData += d })

return finished(fileStream, function (err) {
if (err) { return dataStream.emit('error', err) }
try {
extraProps._shrinkwrap = JSON.parse(srData)
next()
} catch (e) {
dataStream.emit('error', e)
}
})
} else if (needsBin && minimatch(filePath, dirBin + '/**')) {
var relative = path.relative(dirBin, filePath)
if (relative && relative[0] !== '.') {
extraProps.bin[path.basename(relative)] = path.join(dirBin, relative)
}
}
// Drain and get next one
fileStream.on('data', function () {})
next()
})
} else {
extractorStream = through()
}
if (needsShasum) {
shaStream = checksumStream(null, opts.hashAlgorithm)
shaStream.on('digest', function (d) {
extraProps._shasum = d
})
} else {
shaStream = through()
}
// Drain the end stream
extractorStream.on('data', function () {})
return pipe(tarData, shaStream, extractorStream, function (err) {
cb(err, extraProps)
})
}
}
50 changes: 14 additions & 36 deletions lib/registry/manifest.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
'use strict'

var extractShrinkwrap = require('../util/extract-shrinkwrap')
var optCheck = require('../util/opt-check')
var pickManifest = require('./pick-manifest')
var pickRegistry = require('./pick-registry')
var url = require('url')
var request = require('./request')
var tarball = require('./tarball')

module.exports = manifest
function manifest (spec, opts, cb) {
Expand All @@ -28,13 +26,21 @@ function manifest (spec, opts, cb) {
defaultTag: opts.defaultTag
}, function (err, manifest) {
if (err) { return cb(err) }
opts.log.silly(
'registry.manifest',
spec.name + '@' + spec.spec,
'resolved to',
manifest.name + '@' + manifest.version
// Done here instead of ./finalize-manifest because these fields
// have to be filled in differently depending on type.
manifest._shasum = (
manifest.dist && manifest.dist.shasum
) || manifest._shasum
manifest._resolved = (
manifest.dist && manifest.dist.tarball
) || url.resolve(
registry,
'/' + manifest.name +
'/-/' + manifest.name +
'-' + manifest.version +
'.tgz'
)
ensureShrinkwrap(manifest, registry, opts, cb)
cb(null, manifest)
})
})
}
Expand All @@ -45,31 +51,3 @@ function metadataUrl (registry, name) {
: registry
return url.resolve(normalized, name)
}

function ensureShrinkwrap (manifest, registry, opts, cb) {
if (manifest._hasShrinkwrap === false || manifest._shrinkwrap || !manifest.dist) {
opts.log.silly('registry.manifest', 'shrinkwrap extraction not needed')
cb(null, manifest)
} else {
opts.log.silly('registry.manifest', 'extracting shrinkwrap')
opts.memoize = false
var shrinkwrap
var tarstream = tarball.fromManifest(manifest, registry, opts)
extractShrinkwrap(tarstream, opts, function (err, sr) {
if (err) { return cb(err) }
shrinkwrap = sr
})
tarstream.on('data', function () {})
tarstream.on('error', cb)
tarstream.on('end', function () {
manifest._hasShrinkwrap = !!shrinkwrap
manifest._shrinkwrap = shrinkwrap
if (shrinkwrap) {
opts.log.silly('registry.manifest', 'shrinkwrap found')
} else {
opts.log.silly('registry.manifest', 'no shrinkwrap')
}
cb(null, manifest)
})
}
}
29 changes: 17 additions & 12 deletions lib/registry/request.js
Original file line number Diff line number Diff line change
Expand Up @@ -68,19 +68,22 @@ function getStream (uri, registry, opts) {
} else {
opts.log.silly('registry.get', 'trying to grab from cache')
var cs = cache.get.stream(opts.cache, key, opts)
cs.once('data', function () {
opts.log.silly('registry.get', 'got data for', key, 'from cache.')
})
cs.on('metadata', function (meta) {
if (isStale(meta, opts)) {
opts.log.silly(
'registry.get',
'cache data for', key, 'stale. Checking registry.'
)
cs.pause()
switchToRegistry(null, cs, meta)
} else {
opts.log.silly(
'registry.get',
'cached data for', key, 'valid. Reading from cache.'
)
cs.pipe(stream)
}
})
cs.once('error', function (err) {
Expand All @@ -91,7 +94,6 @@ function getStream (uri, registry, opts) {
}
switchToRegistry(err, null)
})
cs.pipe(stream)
stream.on('error', function (e) { cs.emit('error', e) })
stream.on('end', function () { cs.end() })
}
Expand All @@ -104,8 +106,8 @@ function getStream (uri, registry, opts) {
key
)
if (fallback) {
opts.log.silly('registry.get', 'resuming stale cache stream for', key)
fallback.resume()
opts.log.silly('registry.get', 'reading stale cache stream for', key)
fallback.pipe(stream)
} else {
if (!err) {
err = new Error('no offline data available')
Expand All @@ -115,6 +117,7 @@ function getStream (uri, registry, opts) {
}
} else {
if (fallback) {
fallback.pause()
stream.emit('reset')
}
registryStream(
Expand All @@ -123,17 +126,16 @@ function getStream (uri, registry, opts) {
stream.emit('reset')
}).on('cached', function () {
opts.log.silly('registry.get', 'Got a 304. Switching back to cache.')
fallback.resume()
fallback.pipe(stream)
}).once('error', function (err) {
opts.log.silly('registry.get', 'request error: ', err)
if (!fallback) {
stream.emit('error', err)
} else {
opts.log.silly('registry.get', 'using stale', key, 'from cache.')
fallback.resume()
stream.emit('reset')
fallback.pipe(stream)
}
}).on('end', function () {
stream.end()
}).pipe(stream)
}
}
Expand Down Expand Up @@ -189,16 +191,19 @@ function registryStream (key, uri, registry, meta, opts) {
res,
decoder,
to(function (chunk, enc, cb) {
cacheStream.write(chunk, enc, function () {
stream.write(chunk, enc, cb)
})
cacheStream.write(chunk, enc)
stream.write(chunk, enc, cb)
}, function (cb) {
opts.log.silly('registry.get', 'request for', key, 'done.')
cacheStream.end(function () {
opts.log.silly('registry.get', 'wrapped up cacheStream')
stream.end(cb)
})
}),
function () {}
function (err) {
if (err) { return stream.emit('error', err) }
opts.log.silly('registry.get', 'pipeline for', key, 'done')
}
)
} else {
opts.log.silly('registry.get', 'no cache. streaming straight out')
Expand Down

0 comments on commit e51a269

Please sign in to comment.