Skip to content
This repository has been archived by the owner on Jul 3, 2019. It is now read-only.

Commit

Permalink
Merge afd8f24 into 0ca28fe
Browse files Browse the repository at this point in the history
  • Loading branch information
zkat committed Feb 24, 2017
2 parents 0ca28fe + afd8f24 commit a770686
Show file tree
Hide file tree
Showing 19 changed files with 940 additions and 237 deletions.
2 changes: 1 addition & 1 deletion extract.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict'

var cache = require('./lib/cache')
var extractStream = require('./lib/util/extract-stream')
var extractStream = require('./lib/extract-stream')
var pipe = require('mississippi').pipe
var optCheck = require('./lib/util/opt-check')
var rps = require('realize-package-specifier')
Expand Down
45 changes: 21 additions & 24 deletions lib/util/extract-stream.js → lib/extract-stream.js
Original file line number Diff line number Diff line change
@@ -1,18 +1,36 @@
'use strict'

var gunzip = require('./gunzip-maybe')
var gunzip = require('./util/gunzip-maybe')
var path = require('path')
var pipeline = require('mississippi').pipeline
var tar = require('tar-fs')

module.exports = extractStream
function extractStream (dest, opts) {
opts = opts || {}
var sawIgnores = {}
return pipeline(gunzip(), tar.extract(dest, {
map: function (header) {
if (process.platform !== 'win32') {
header.uid = opts.uid == null ? header.uid : opts.uid
header.gid = opts.gid == null ? header.gid : opts.gid
}
// Note: This mirrors logic in the fs read operations that are
// employed during tarball creation, in the fstream-npm module.
// It is duplicated here to handle tarballs that are created
// using other means, such as system tar or git archive.
if (header.type === 'file') {
var base = path.basename(header.name)
if (base === '.npmignore') {
sawIgnores[header.name] = true
} else if (base === '.gitignore') {
var npmignore = header.name.replace(/\.gitignore$/, '.npmignore')
if (!sawIgnores[npmignore]) {
// Rename, may be clobbered later.
header.name = npmignore
}
}
}
return header
},
ignore: makeIgnore(opts.log),
Expand All @@ -31,36 +49,15 @@ function makeIgnore (log) {
}

function _ignore (name, header, sawIgnores, logger) {
if (header.type.match(/^.*Link$/)) {
if (header.type.match(/^.*link$/)) {
if (logger) {
logger.warn(
'extract-stream',
'excluding symbolic link',
header.path, '->', header.linkname)
header.name, '->', header.linkname)
}
return true
}

// Note: This mirrors logic in the fs read operations that are
// employed during tarball creation, in the fstream-npm module.
// It is duplicated here to handle tarballs that are created
// using other means, such as system tar or git archive.
if (header.type === 'File') {
var base = path.basename(header.path)
if (base === '.npmignore') {
sawIgnores[header.path] = true
} else if (base === '.gitignore') {
var npmignore = header.path.replace(/\.gitignore$/, '.npmignore')
if (sawIgnores[npmignore]) {
// Skip this one, already seen.
return true
} else {
// Rename, may be clobbered later.
header.path = npmignore
header._path = npmignore
}
}
}

return false
}
154 changes: 154 additions & 0 deletions lib/finalize-manifest.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
'use strict'

var checksumStream = require('checksum-stream')
var dezalgo = require('dezalgo')
var finished = require('mississippi').finished
var gunzip = require('./util/gunzip-maybe')
var minimatch = require('minimatch')
var normalize = require('normalize-package-data')
var optCheck = require('./util/opt-check')
var path = require('path')
var pipe = require('mississippi').pipe
var pipeline = require('mississippi').pipeline
var tar = require('tar-stream')
var through = require('mississippi').through

module.exports = finalizeManifest
function finalizeManifest (pkg, spec, opts, cb) {
tarballedProps(pkg, spec, opts, function (err, props) {
if (err) { return cb(err) }
// normalize should not add any fields, and once
// makeManifest completes, it should never be modified.
cb(null, new Manifest(pkg, props))
})
}

module.exports.Manifest = Manifest
function Manifest (pkg, fromTarball) {
fromTarball = fromTarball || {}
this.name = pkg.name
this.version = pkg.version
this.dependencies = pkg.dependencies || {}
this.optionalDependencies = pkg.optionalDependencies || {}
this.devDependencies = pkg.devDependencies || {}
var bundled = (
pkg.bundledDependencies ||
pkg.bundleDependencies ||
false
)
this.bundleDependencies = bundled
this.peerDependencies = pkg.peerDependencies || {}

// This one depends entirely on each handler.
this._resolved = pkg._resolved

// Not all handlers (or registries) provide these out of the box,
// and if they don't, we need to extract and read the tarball ourselves.
// These are details required by the installer.
this._shasum = pkg._shasum || fromTarball._shasum
this._shrinkwrap = pkg._shrinkwrap || fromTarball._shrinkwrap || null
this.bin = pkg.bin || fromTarball.bin || null

if (this.bin && Array.isArray(this.bin)) {
// Code yanked from read-package-json.
var m = pkg.directories && pkg.directories.bin || '.'
this.bin = this.bin.reduce(function (acc, mf) {
if (mf && mf.charAt(0) !== '.') {
var f = path.basename(mf)
acc[f] = path.join(m, mf)
}
return acc
}, {})
}

this._id = null // filled in by normalize-package-data, but unnecessary

// TODO - freezing and inextensibility pending npm changes. See test suite.
// Object.preventExtensions(this)
normalize(this)

// I don't want this why did you give it to me. Go away. 🔥🔥🔥🔥
delete this.readme

// Object.freeze(this)
}

// Some things aren't filled in by standard manifest fetching.
// If this function needs to do its work, it will grab the
// package tarball, extract it, and take whatever it needs
// from the stream.
function tarballedProps (pkg, spec, opts, cb) {
cb = dezalgo(cb)
var extraProps = {}
var needsShrinkwrap = (
pkg._hasShrinkwrap !== false &&
!pkg._shrinkwrap
)
var needsBin = !!(
!pkg.bin &&
pkg.directories &&
pkg.directories.bin
)
var needsShasum = !pkg._shasum
if (!needsShrinkwrap && !needsBin && !needsShasum) {
opts.log.silly('finalize-manifest', 'Skipping tarball extraction -- nothing needed.')
return cb(null, extraProps)
} else {
opts = optCheck(opts)
opts.memoize = false
var tarball = require('./handlers/' + spec.type + '/tarball')
var tarData = tarball.fromManifest(pkg, spec, opts)
var shaStream = null
var extractorStream = null

if (needsShrinkwrap || needsBin) {
opts.log.silly('finalize-manifest', 'parsing tarball for', spec.name)
var dirBin = pkg.directories && pkg.directories.bin
extraProps.bin = pkg.bin || {}
var dataStream = tar.extract()
extractorStream = pipeline(gunzip(), dataStream)
dataStream.on('entry', function doEntry (header, fileStream, next) {
var filePath = header.name.replace(/[^/]+\//, '')
if (needsShrinkwrap && filePath === 'npm-shrinkwrap.json') {
var srData = ''
fileStream.on('data', function (d) { srData += d })

return finished(fileStream, function (err) {
if (err) { return dataStream.emit('error', err) }
try {
extraProps._shrinkwrap = JSON.parse(srData)
next()
} catch (e) {
dataStream.emit('error', e)
}
})
} else if (needsBin && minimatch(filePath, dirBin + '/**')) {
var relative = path.relative(dirBin, filePath)
if (relative && relative[0] !== '.') {
extraProps.bin[path.basename(relative)] = path.join(dirBin, relative)
}
}
// Drain and get next one
fileStream.on('data', function () {})
next()
})
} else {
extractorStream = through()
}
if (needsShasum) {
shaStream = checksumStream({
algorithm: opts.hashAlgorithm
})
shaStream.on('digest', function (d) {
extraProps._shasum = d
})
} else {
shaStream = through()
}
// Drain the end stream
extractorStream.on('data', function () {})
return pipe(tarData, shaStream, extractorStream, function (err) {
cb(err, extraProps)
})
}
}
50 changes: 14 additions & 36 deletions lib/registry/manifest.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
'use strict'

var extractShrinkwrap = require('../util/extract-shrinkwrap')
var optCheck = require('../util/opt-check')
var pickManifest = require('./pick-manifest')
var pickRegistry = require('./pick-registry')
var url = require('url')
var request = require('./request')
var tarball = require('./tarball')

module.exports = manifest
function manifest (spec, opts, cb) {
Expand All @@ -28,13 +26,21 @@ function manifest (spec, opts, cb) {
defaultTag: opts.defaultTag
}, function (err, manifest) {
if (err) { return cb(err) }
opts.log.silly(
'registry.manifest',
spec.name + '@' + spec.spec,
'resolved to',
manifest.name + '@' + manifest.version
// Done here instead of ./finalize-manifest because these fields
// have to be filled in differently depending on type.
manifest._shasum = (
manifest.dist && manifest.dist.shasum
) || manifest._shasum
manifest._resolved = (
manifest.dist && manifest.dist.tarball
) || url.resolve(
registry,
'/' + manifest.name +
'/-/' + manifest.name +
'-' + manifest.version +
'.tgz'
)
ensureShrinkwrap(manifest, registry, opts, cb)
cb(null, manifest)
})
})
}
Expand All @@ -45,31 +51,3 @@ function metadataUrl (registry, name) {
: registry
return url.resolve(normalized, name)
}

function ensureShrinkwrap (manifest, registry, opts, cb) {
if (manifest._hasShrinkwrap === false || manifest._shrinkwrap || !manifest.dist) {
opts.log.silly('registry.manifest', 'shrinkwrap extraction not needed')
cb(null, manifest)
} else {
opts.log.silly('registry.manifest', 'extracting shrinkwrap')
opts.memoize = false
var shrinkwrap
var tarstream = tarball.fromManifest(manifest, registry, opts)
extractShrinkwrap(tarstream, opts, function (err, sr) {
if (err) { return cb(err) }
shrinkwrap = sr
})
tarstream.on('data', function () {})
tarstream.on('error', cb)
tarstream.on('end', function () {
manifest._hasShrinkwrap = !!shrinkwrap
manifest._shrinkwrap = shrinkwrap
if (shrinkwrap) {
opts.log.silly('registry.manifest', 'shrinkwrap found')
} else {
opts.log.silly('registry.manifest', 'no shrinkwrap')
}
cb(null, manifest)
})
}
}
10 changes: 5 additions & 5 deletions lib/registry/tarball.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ var through = require('mississippi').through
module.exports = tarball
function tarball (spec, opts) {
opts = optCheck(opts)
var registry = pickRegistry(spec, opts)
opts.log.verbose(
'registry.tarball',
'looking up registry-based metadata for ', spec
Expand All @@ -23,17 +22,18 @@ function tarball (spec, opts) {
'registry metadata found. Downloading ', manifest.name + '@' + manifest.version
)
pipe(
fromManifest(manifest, registry),
fromManifest(manifest, spec, opts),
stream
)
})
return stream
}

module.exports.fromManifest = fromManifest
function fromManifest (manifest, registry, opts) {
function fromManifest (manifest, spec, opts) {
opts = optCheck(opts)
var uri = manifest.dist && manifest.dist.tarball
opts.digest = manifest.dist && manifest.dist.shasum
var registry = pickRegistry(spec, opts)
var uri = manifest._resolved
opts.digest = manifest._shasum
return request.stream(uri, registry, opts)
}
Loading

0 comments on commit a770686

Please sign in to comment.