Permalink
Browse files

Make tarballs using fstream-npm

  • Loading branch information...
1 parent e220ba7 commit 41daac7ac6535191fdfabd0c045adac35183b8bb @isaacs isaacs committed Mar 27, 2012
Showing with 19 additions and 436 deletions.
  1. +19 −436 lib/utils/tar.js
View
455 lib/utils/tar.js
@@ -1,27 +1,21 @@
-// XXX lib/cache.js and this file need to be rewritten.
-
// commands for packing and unpacking tarballs
// this file is used by lib/cache.js
var npm = require("../npm.js")
, fs = require("graceful-fs")
- , exec = require("./exec.js")
- , find = require("./find.js")
- , mkdir = require("./mkdir-p.js")
- , asyncMap = require("slide").asyncMap
, path = require("path")
, log = require("./log.js")
, uidNumber = require("./uid-number.js")
, rm = require("rimraf")
, readJson = require("./read-json.js")
, relativize = require("./relativize.js")
, cache = require("../cache.js")
- , excludes = require("./excludes.js")
, myUid = process.getuid && process.getuid()
, myGid = process.getgid && process.getgid()
, tar = require("tar")
, zlib = require("zlib")
, fstream = require("fstream")
+ , Packer = require("fstream-npm")
if (process.env.SUDO_UID && myUid === 0) {
if (!isNaN(process.env.SUDO_UID)) myUid = +process.env.SUDO_UID
@@ -30,100 +24,16 @@ if (process.env.SUDO_UID && myUid === 0) {
exports.pack = pack
exports.unpack = unpack
-exports.makeList = makeList
function pack (targetTarball, folder, pkg, dfc, cb) {
+ log.verbose([targetTarball, folder], "tar.pack")
if (typeof cb !== "function") cb = dfc, dfc = true
- folder = path.resolve(folder)
-
- log.verbose(folder, "pack")
-
- if (typeof pkg === "function") {
- cb = pkg, pkg = null
- return readJson(path.resolve(folder, "package.json"), function (er, pkg) {
- if (er) return log.er(cb, "Couldn't find package.json in "+folder)(er)
- pack(targetTarball, folder, pkg, dfc, cb)
- })
- }
- log.verbose(folder+" "+targetTarball, "pack")
- var parent = path.dirname(folder)
- , addFolder = path.basename(folder)
-
- var confEx = npm.config.get("ignore")
- log.silly(folder, "makeList")
- makeList(folder, pkg, dfc, function (er, files, cleanup) {
- if (er) return cb(er)
- // log.silly(files, "files")
- return packFiles(targetTarball, parent, files, pkg, function (er) {
- if (!cleanup || !cleanup.length) return cb(er)
- // try to be a good citizen, even/especially in the event of failure.
- cleanupResolveLinkDep(cleanup, function (er2) {
- if (er || er2) {
- if (er) log(er, "packing tarball")
- if (er2) log(er2, "while cleaning up resolved deps")
- }
- return cb(er || er2)
- })
- })
- })
-}
-
-function packFiles (targetTarball, parent, files, pkg, cb_) {
-
- var p
-
- files = files.map(function (f) {
- p = f.split(/\/|\\/)[0]
- return path.resolve(parent, f)
- })
-
- parent = path.resolve(parent, p)
-
- var called = false
- function cb (er) {
- if (called) return
- called = true
- cb_(er)
- }
log.verbose(targetTarball, "tarball")
- log.verbose(parent, "parent")
- fstream.Reader({ type: "Directory"
- , path: parent
- , filter: function () {
- // files should *always* get into tarballs
- // in a user-writable state, even if they're
- // being installed from some wackey vm-mounted
- // read-only filesystem.
- this.props.mode = this.props.mode | 0200
- var inc = -1 !== files.indexOf(this.path)
-
- // symbolic links are not allowed in packages.
- if (this.type.match(/^.*Link$/)) {
- log.warn( this.path.substr(parent.length + 1)
- + ' -> ' + this.linkpath
- , "excluding symbolic link")
- return false
- }
-
- // WARNING! Hackety hack!
- // XXX Fix this in a better way.
- // Rename .gitignore to .npmignore if there is not a
- // .npmignore file there already, the better to lock
- // down installed packages with git for deployment.
- if (this.basename === ".gitignore") {
- if (this.parent.entries.indexOf(".npmignore") !== -1) {
- return false
- }
- var d = path.dirname(this.path)
- this.basename = ".npmignore"
- this.path = path.join(d, ".npmignore")
- }
+ log.verbose(folder, "folder")
+ new Packer({ path: folder, type: "Directory", isDirectory: true })
+ .on("error", log.er(cb, "error reading "+folder))
- return inc
- }
- })
- .on("error", log.er(cb, "error reading "+parent))
// By default, npm includes some proprietary attributes in the
// package tarball. This is sane, and allowed by the spec.
// However, npm *itself* excludes these from its own package,
@@ -135,11 +45,14 @@ function packFiles (targetTarball, parent, files, pkg, cb_) {
.on("error", log.er(cb, "gzip error "+targetTarball))
.pipe(fstream.Writer({ type: "File", path: targetTarball }))
.on("error", log.er(cb, "Could not write "+targetTarball))
- .on("close", cb)
+ .on("close", function () {
+ cb()
+ })
}
function unpack (tarball, unpackTarget, dMode, fMode, uid, gid, cb) {
+ log.verbose(tarball, "unpack")
if (typeof cb !== "function") cb = gid, gid = null
if (typeof cb !== "function") cb = uid, uid = null
if (typeof cb !== "function") cb = fMode, fMode = npm.modes.file
@@ -161,23 +74,16 @@ function unpack_ ( tarball, unpackTarget, dMode, fMode, uid, gid, cb ) {
rm(unpackTarget, function (er) {
if (er) return cb(er)
- mkdir(unpackTarget, dMode || npm.modes.exec, uid, gid, function (er) {
- log.verbose([uid, gid], "unpack_ uid, gid")
- log.verbose(unpackTarget, "unpackTarget")
+ // cp the gzip of the tarball, pipe the stdout into tar's stdin
+ // gzip {tarball} --decompress --stdout \
+ // | tar -mvxpf - --strip-components=1 -C {unpackTarget}
+ gunzTarPerm( tarball, unpackTarget
+ , dMode, fMode
+ , uid, gid
+ , function (er, folder) {
if (er) return cb(er)
-
- // cp the gzip of the tarball, pipe the stdout into tar's stdin
- // gzip {tarball} --decompress --stdout \
- // | tar -mvxpf - --strip-components=1 -C {unpackTarget}
- gunzTarPerm( tarball, unpackTarget
- , dMode, fMode
- , uid, gid
- , function (er, folder) {
- if (er) return cb(er)
- log.verbose(folder, "gunzed")
- readJson(path.resolve(folder, "package.json"), cb)
- })
- })
+ readJson(path.resolve(folder, "package.json"), cb)
+ })
})
}
@@ -204,6 +110,7 @@ function gunzTarPerm (tarball, target, dMode, fMode, uid, gid, cb_) {
}
function extractEntry (entry) {
+ log.silly(entry.path, "extracting entry")
// never create things that are user-unreadable,
// or dirs that are user-un-listable. Only leads to headaches.
var originalMode = entry.mode = entry.mode || entry.props.mode
@@ -283,327 +190,3 @@ function gunzTarPerm (tarball, target, dMode, fMode, uid, gid, cb_) {
fst.emit("data", c)
})
}
-
-function makeList (dir, pkg, dfc, cb) {
- if (typeof cb !== "function") cb = dfc, dfc = true
- if (typeof cb !== "function") cb = pkg, pkg = null
- dir = path.resolve(dir)
-
- if (!pkg.path) pkg.path = dir
-
- var name = path.basename(dir)
-
- // since this is a top-level traversal, get the user and global
- // exclude files, as well as the "ignore" config setting.
- var confIgnore = npm.config.get("ignore").trim()
- .split(/[\n\r\s\t]+/)
- .filter(function (i) { return i.trim() })
- , userIgnore = npm.config.get("userignorefile")
- , globalIgnore = npm.config.get("globalignorefile")
- , userExclude
- , globalExclude
-
- confIgnore.dir = dir
- confIgnore.name = "confIgnore"
-
- var defIgnore = ["build/"]
- defIgnore.dir = dir
-
- // TODO: only look these up once, and cache outside this function
- excludes.parseIgnoreFile( userIgnore, null, dir
- , function (er, uex) {
- if (er) return cb(er)
- userExclude = uex
- next()
- })
-
- excludes.parseIgnoreFile( globalIgnore, null, dir
- , function (er, gex) {
- if (er) return cb(er)
- globalExclude = gex
- next()
- })
-
- function next () {
- if (!globalExclude || !userExclude) return
- var exList = [ defIgnore, confIgnore, globalExclude, userExclude ]
-
- makeList_(dir, pkg, exList, dfc, function (er, files, cleanup) {
- if (er) return cb(er)
- var dirLen = dir.replace(/(\/|\\)$/, "").length + 1
- log.silly([dir, dirLen], "dir, dirLen")
- files = files.map(function (file) {
- return path.join(name, file.substr(dirLen))
- })
- return cb(null, files, cleanup)
- })
- }
-}
-
-// Patterns ending in slashes will only match targets
-// ending in slashes. To implement this, add a / to
-// the filename iff it lstats isDirectory()
-function readDir (dir, pkg, dfc, cb) {
- fs.readdir(dir, function (er, files) {
- if (er) return cb(er)
- files = files.filter(function (f) {
- return f && f.charAt(0) !== "/" && f.indexOf("\0") === -1
- })
- asyncMap(files, function (file, cb) {
- fs.lstat(path.resolve(dir, file), function (er, st) {
- if (er) return cb(null, [])
- // if it's a directory, then tack "/" onto the name
- // so that it can match dir-only patterns in the
- // include/exclude logic later.
- if (st.isDirectory()) return cb(null, file + "/")
-
- // if it's a symlink, then we need to do some more
- // complex stuff for GH-691
- if (st.isSymbolicLink()) return readSymlink(dir, file, pkg, dfc, cb)
-
- // otherwise, just let it on through.
- return cb(null, file)
- })
- }, cb)
- })
-}
-
-// just see where this link is pointing, and resolve relative paths.
-function shallowReal (link, cb) {
- link = path.resolve(link)
- fs.readlink(link, function (er, t) {
- if (er) return cb(er)
- return cb(null, path.resolve(path.dirname(link), t), t)
- })
-}
-
-function readSymlink (dir, file, pkg, dfc, cb) {
- var isNM = dfc
- && path.basename(dir) === "node_modules"
- && path.dirname(dir) === pkg.path
- // see if this thing is pointing outside of the package.
- // external symlinks are resolved for deps, ignored for other things.
- // internal symlinks are allowed through.
- var df = path.resolve(dir, file)
- shallowReal(df, function (er, r, target) {
- if (er) return cb(null, []) // wtf? exclude file.
- if (r.indexOf(dir) === 0) return cb(null, file) // internal
- if (!isNM) return cb(null, []) // external non-dep
- // now the fun stuff!
- fs.realpath(df, function (er, resolved) {
- if (er) return cb(null, []) // can't add it.
- readJson(path.resolve(resolved, "package.json"), function (er) {
- if (er) return cb(null, []) // not a package
- resolveLinkDep(dir, file, resolved, target, pkg, function (er, f, c) {
- cb(er, f, c)
- })
- })
- })
- })
-}
-
-// put the link back the way it was.
-function cleanupResolveLinkDep (cleanup, cb) {
- // cut it out of the list, so that cycles will be broken.
- if (!cleanup) return cb()
-
- asyncMap(cleanup, function (d, cb) {
- rm(d[1], function (er) {
- if (er) return cb(er)
- fs.symlink(d[0], d[1], cb)
- })
- }, cb)
-}
-
-function resolveLinkDep (dir, file, resolved, target, pkg, cb) {
- // we've already decided that this is a dep that will be bundled.
- // make sure the data reflects this.
- var bd = pkg.bundleDependencies || pkg.bundledDependencies || []
- delete pkg.bundledDependencies
- pkg.bundleDependencies = bd
- var f = path.resolve(dir, file)
- , cleanup = [[target, f, resolved]]
-
- if (bd.indexOf(file) === -1) {
- // then we don't do this one.
- // just move the symlink out of the way.
- return rm(f, function (er) {
- cb(er, file, cleanup)
- })
- }
-
- rm(f, function (er) {
- if (er) return cb(er)
- cache.add(resolved, function (er, data) {
- if (er) return cb(er)
- cache.unpack(data.name, data.version, f, function (er, data) {
- if (er) return cb(er)
- // now clear out the cache entry, since it's weird, probably.
- // pass the cleanup object along so that the thing getting the
- // list of files knows what to clean up afterwards.
- cache.clean([data._id], function (er) { cb(er, file, cleanup) })
- })
- })
- })
-}
-
-// exList is a list of ignore lists.
-// Each exList item is an array of patterns of files to ignore
-//
-function makeList_ (dir, pkg, exList, dfc, cb) {
- var files = null
- , cleanup = null
-
- readDir(dir, pkg, dfc, function (er, f, c) {
- if (er) return cb(er)
- cleanup = c
- files = f.map(function (f) {
- // no nulls in paths!
- return f.split(/\0/)[0]
- }).filter(function (f) {
- // always remove all source control folders and
- // waf/vim/OSX garbage. this is a firm requirement.
- return !( f === ".git/"
- || f === ".lock-wscript"
- || f.match(/^\.wafpickle-[0-9]+$/)
- || f === "CVS/"
- || f === ".svn/"
- || f === ".hg/"
- || f.match(/^\..*\.swp/)
- || f === ".DS_Store"
- || f.match(/^\._/)
- || f === "npm-debug.log"
- || f === ""
- || f.charAt(0) === "/"
- )
- })
-
- // if (files.length > 0) files.push(".")
-
- if (files.indexOf("package.json") !== -1 && dir !== pkg.path) {
- // a package.json file starts the whole exclude/include
- // logic all over. Otherwise, a parent could break its
- // deps with its files list or .npmignore file.
- readJson(path.resolve(dir, "package.json"), function (er, data) {
- if (!er && typeof data === "object") {
- data.path = dir
- return makeList(dir, data, dfc, function (er, files) {
- // these need to be mounted onto the directory now.
- cb(er, files && files.map(function (f) {
- return path.resolve(path.dirname(dir), f)
- }))
- })
- }
- next()
- })
- //next()
- } else next()
-
- // add a local ignore file, if found.
- if (files.indexOf(".npmignore") === -1
- && files.indexOf(".gitignore") === -1) next()
- else {
- excludes.addIgnoreFile( path.resolve(dir, ".npmignore")
- , ".gitignore"
- , exList
- , dir
- , function (er, list) {
- if (!er) exList = list
- next(er)
- })
- }
- })
-
- var n = 2
- , errState = null
- function next (er) {
- if (errState) return
- if (er) return cb(errState = er, [], cleanup)
- if (-- n > 0) return
-
- if (!pkg) return cb(new Error("No package.json file in "+dir))
- if (pkg.path === dir && pkg.files) {
- pkg.files = pkg.files.filter(function (f) {
- f = f.trim()
- return f && f.charAt(0) !== "#"
- })
- if (!pkg.files.length) pkg.files = null
- }
- if (pkg.path === dir && pkg.files) {
- // stuff on the files list MUST be there.
- // ignore everything, then include the stuff on the files list.
- var pkgFiles = ["*"].concat(pkg.files.map(function (f) {
- return "!" + f
- }))
- pkgFiles.dir = dir
- pkgFiles.packageFiles = true
- exList.push(pkgFiles)
- }
-
- if (path.basename(dir) === "node_modules"
- && pkg.path === path.dirname(dir)
- // do fancy crap
- && dfc
- // not already part of a bundled dependency
- && (path.basename(path.dirname(pkg.path)) !== "node_modules"
- // unless it's the root
- || pkg.path === npm.prefix)) {
- log.verbose(dir, "doing fancy crap")
- files = filterNodeModules(files, pkg)
- } else {
- // If a directory is excluded, we still need to be
- // able to *include* a file within it, and have that override
- // the prior exclusion.
- //
- // This whole makeList thing probably needs to be rewritten
- files = files.filter(function (f) {
- return excludes.filter(dir, exList)(f) || f.slice(-1) === "/"
- })
- }
-
-
- asyncMap(files, function (file, cb) {
- // if this is a dir, then dive into it.
- // otherwise, don't.
- file = path.resolve(dir, file)
-
- // in 0.6.0, fs.readdir can produce some really odd results.
- // XXX: remove this and make the engines hash exclude 0.6.0
- if (file.indexOf(dir) !== 0) {
- return cb(null, [])
- }
-
- fs.lstat(file, function (er, st) {
- if (er) return cb(er)
- if (st.isDirectory()) {
- return makeList_(file, pkg, exList, dfc, cb)
- }
- return cb(null, file)
- })
- }, function (er, files, c) {
- if (c) cleanup = (cleanup || []).concat(c)
- if (files.length > 0) files.push(dir)
- return cb(er, files, cleanup)
- })
- }
-}
-
-// only include node_modules folder that are:
-// 1. not on the dependencies list or
-// 2. on the "bundleDependencies" list.
-function filterNodeModules (files, pkg) {
- var bd = pkg.bundleDependencies || pkg.bundledDependencies || []
- , deps = Object.keys(pkg.dependencies || {})
- .filter(function (key) { return !pkg.dependencies[key].extraneous })
- .concat(Object.keys(pkg.devDependencies || {}))
-
- delete pkg.bundledDependencies
- pkg.bundleDependencies = bd
-
- return files.filter(function (f) {
- f = f.replace(/\/$/, "")
- return f.charAt(0) !== "."
- && f.charAt(0) !== "_"
- && bd.indexOf(f) !== -1
- })
-}

0 comments on commit 41daac7

Please sign in to comment.