Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Make tarballs using fstream-npm

  • Loading branch information...
commit 41daac7ac6535191fdfabd0c045adac35183b8bb 1 parent e220ba7
Isaac Z. Schlueter authored March 26, 2012

Showing 1 changed file with 19 additions and 436 deletions. Show diff stats Hide diff stats

  1. 455  lib/utils/tar.js
455  lib/utils/tar.js
... ...
@@ -1,14 +1,8 @@
1  
-// XXX lib/cache.js and this file need to be rewritten.
2  
-
3 1
 // commands for packing and unpacking tarballs
4 2
 // this file is used by lib/cache.js
5 3
 
6 4
 var npm = require("../npm.js")
7 5
   , fs = require("graceful-fs")
8  
-  , exec = require("./exec.js")
9  
-  , find = require("./find.js")
10  
-  , mkdir = require("./mkdir-p.js")
11  
-  , asyncMap = require("slide").asyncMap
12 6
   , path = require("path")
13 7
   , log = require("./log.js")
14 8
   , uidNumber = require("./uid-number.js")
@@ -16,12 +10,12 @@ var npm = require("../npm.js")
16 10
   , readJson = require("./read-json.js")
17 11
   , relativize = require("./relativize.js")
18 12
   , cache = require("../cache.js")
19  
-  , excludes = require("./excludes.js")
20 13
   , myUid = process.getuid && process.getuid()
21 14
   , myGid = process.getgid && process.getgid()
22 15
   , tar = require("tar")
23 16
   , zlib = require("zlib")
24 17
   , fstream = require("fstream")
  18
+  , Packer = require("fstream-npm")
25 19
 
26 20
 if (process.env.SUDO_UID && myUid === 0) {
27 21
   if (!isNaN(process.env.SUDO_UID)) myUid = +process.env.SUDO_UID
@@ -30,100 +24,16 @@ if (process.env.SUDO_UID && myUid === 0) {
30 24
 
31 25
 exports.pack = pack
32 26
 exports.unpack = unpack
33  
-exports.makeList = makeList
34 27
 
35 28
 function pack (targetTarball, folder, pkg, dfc, cb) {
  29
+  log.verbose([targetTarball, folder], "tar.pack")
36 30
   if (typeof cb !== "function") cb = dfc, dfc = true
37  
-  folder = path.resolve(folder)
38  
-
39  
-  log.verbose(folder, "pack")
40  
-
41  
-  if (typeof pkg === "function") {
42  
-    cb = pkg, pkg = null
43  
-    return readJson(path.resolve(folder, "package.json"), function (er, pkg) {
44  
-      if (er) return log.er(cb, "Couldn't find package.json in "+folder)(er)
45  
-      pack(targetTarball, folder, pkg, dfc, cb)
46  
-    })
47  
-  }
48  
-  log.verbose(folder+" "+targetTarball, "pack")
49  
-  var parent = path.dirname(folder)
50  
-    , addFolder = path.basename(folder)
51  
-
52  
-  var confEx = npm.config.get("ignore")
53  
-  log.silly(folder, "makeList")
54  
-  makeList(folder, pkg, dfc, function (er, files, cleanup) {
55  
-    if (er) return cb(er)
56  
-    // log.silly(files, "files")
57  
-    return packFiles(targetTarball, parent, files, pkg, function (er) {
58  
-      if (!cleanup || !cleanup.length) return cb(er)
59  
-      // try to be a good citizen, even/especially in the event of failure.
60  
-      cleanupResolveLinkDep(cleanup, function (er2) {
61  
-        if (er || er2) {
62  
-          if (er) log(er, "packing tarball")
63  
-          if (er2) log(er2, "while cleaning up resolved deps")
64  
-        }
65  
-        return cb(er || er2)
66  
-      })
67  
-    })
68  
-  })
69  
-}
70  
-
71  
-function packFiles (targetTarball, parent, files, pkg, cb_) {
72  
-
73  
-  var p
74  
-
75  
-  files = files.map(function (f) {
76  
-    p = f.split(/\/|\\/)[0]
77  
-    return path.resolve(parent, f)
78  
-  })
79  
-
80  
-  parent = path.resolve(parent, p)
81  
-
82  
-  var called = false
83  
-  function cb (er) {
84  
-    if (called) return
85  
-    called = true
86  
-    cb_(er)
87  
-  }
88 31
 
89 32
   log.verbose(targetTarball, "tarball")
90  
-  log.verbose(parent, "parent")
91  
-  fstream.Reader({ type: "Directory"
92  
-                 , path: parent
93  
-                 , filter: function () {
94  
-                     // files should *always* get into tarballs
95  
-                     // in a user-writable state, even if they're
96  
-                     // being installed from some wackey vm-mounted
97  
-                     // read-only filesystem.
98  
-                     this.props.mode = this.props.mode | 0200
99  
-                     var inc = -1 !== files.indexOf(this.path)
100  
-
101  
-                     // symbolic links are not allowed in packages.
102  
-                     if (this.type.match(/^.*Link$/)) {
103  
-                       log.warn( this.path.substr(parent.length + 1)
104  
-                               + ' -> ' + this.linkpath
105  
-                               , "excluding symbolic link")
106  
-                       return false
107  
-                     }
108  
-
109  
-                     // WARNING! Hackety hack!
110  
-                     // XXX Fix this in a better way.
111  
-                     // Rename .gitignore to .npmignore if there is not a
112  
-                     // .npmignore file there already, the better to lock
113  
-                     // down installed packages with git for deployment.
114  
-                     if (this.basename === ".gitignore") {
115  
-                       if (this.parent.entries.indexOf(".npmignore") !== -1) {
116  
-                         return false
117  
-                       }
118  
-                       var d = path.dirname(this.path)
119  
-                       this.basename = ".npmignore"
120  
-                       this.path = path.join(d, ".npmignore")
121  
-                     }
  33
+  log.verbose(folder, "folder")
  34
+  new Packer({ path: folder, type: "Directory", isDirectory: true })
  35
+    .on("error", log.er(cb, "error reading "+folder))
122 36
 
123  
-                     return inc
124  
-                   }
125  
-                 })
126  
-    .on("error", log.er(cb, "error reading "+parent))
127 37
     // By default, npm includes some proprietary attributes in the
128 38
     // package tarball.  This is sane, and allowed by the spec.
129 39
     // However, npm *itself* excludes these from its own package,
@@ -135,11 +45,14 @@ function packFiles (targetTarball, parent, files, pkg, cb_) {
135 45
     .on("error", log.er(cb, "gzip error "+targetTarball))
136 46
     .pipe(fstream.Writer({ type: "File", path: targetTarball }))
137 47
     .on("error", log.er(cb, "Could not write "+targetTarball))
138  
-    .on("close", cb)
  48
+    .on("close", function () {
  49
+      cb()
  50
+    })
139 51
 }
140 52
 
141 53
 
142 54
 function unpack (tarball, unpackTarget, dMode, fMode, uid, gid, cb) {
  55
+  log.verbose(tarball, "unpack")
143 56
   if (typeof cb !== "function") cb = gid, gid = null
144 57
   if (typeof cb !== "function") cb = uid, uid = null
145 58
   if (typeof cb !== "function") cb = fMode, fMode = npm.modes.file
@@ -161,23 +74,16 @@ function unpack_ ( tarball, unpackTarget, dMode, fMode, uid, gid, cb ) {
161 74
   rm(unpackTarget, function (er) {
162 75
     if (er) return cb(er)
163 76
 
164  
-    mkdir(unpackTarget, dMode || npm.modes.exec, uid, gid, function (er) {
165  
-      log.verbose([uid, gid], "unpack_ uid, gid")
166  
-      log.verbose(unpackTarget, "unpackTarget")
  77
+    // cp the gzip of the tarball, pipe the stdout into tar's stdin
  78
+    // gzip {tarball} --decompress --stdout \
  79
+    //   | tar -mvxpf - --strip-components=1 -C {unpackTarget}
  80
+    gunzTarPerm( tarball, unpackTarget
  81
+               , dMode, fMode
  82
+               , uid, gid
  83
+               , function (er, folder) {
167 84
       if (er) return cb(er)
168  
-
169  
-      // cp the gzip of the tarball, pipe the stdout into tar's stdin
170  
-      // gzip {tarball} --decompress --stdout \
171  
-      //   | tar -mvxpf - --strip-components=1 -C {unpackTarget}
172  
-      gunzTarPerm( tarball, unpackTarget
173  
-                 , dMode, fMode
174  
-                 , uid, gid
175  
-                 , function (er, folder) {
176  
-        if (er) return cb(er)
177  
-        log.verbose(folder, "gunzed")
178  
-        readJson(path.resolve(folder, "package.json"), cb)
179  
-     })
180  
-    })
  85
+      readJson(path.resolve(folder, "package.json"), cb)
  86
+   })
181 87
   })
182 88
 }
183 89
 
@@ -204,6 +110,7 @@ function gunzTarPerm (tarball, target, dMode, fMode, uid, gid, cb_) {
204 110
   }
205 111
 
206 112
   function extractEntry (entry) {
  113
+    log.silly(entry.path, "extracting entry")
207 114
     // never create things that are user-unreadable,
208 115
     // or dirs that are user-un-listable. Only leads to headaches.
209 116
     var originalMode = entry.mode = entry.mode || entry.props.mode
@@ -283,327 +190,3 @@ function gunzTarPerm (tarball, target, dMode, fMode, uid, gid, cb_) {
283 190
     fst.emit("data", c)
284 191
   })
285 192
 }
286  
-
287  
-function makeList (dir, pkg, dfc, cb) {
288  
-  if (typeof cb !== "function") cb = dfc, dfc = true
289  
-  if (typeof cb !== "function") cb = pkg, pkg = null
290  
-  dir = path.resolve(dir)
291  
-
292  
-  if (!pkg.path) pkg.path = dir
293  
-
294  
-  var name = path.basename(dir)
295  
-
296  
-  // since this is a top-level traversal, get the user and global
297  
-  // exclude files, as well as the "ignore" config setting.
298  
-  var confIgnore = npm.config.get("ignore").trim()
299  
-        .split(/[\n\r\s\t]+/)
300  
-        .filter(function (i) { return i.trim() })
301  
-    , userIgnore = npm.config.get("userignorefile")
302  
-    , globalIgnore = npm.config.get("globalignorefile")
303  
-    , userExclude
304  
-    , globalExclude
305  
-
306  
-  confIgnore.dir = dir
307  
-  confIgnore.name = "confIgnore"
308  
-
309  
-  var defIgnore = ["build/"]
310  
-  defIgnore.dir = dir
311  
-
312  
-  // TODO: only look these up once, and cache outside this function
313  
-  excludes.parseIgnoreFile( userIgnore, null, dir
314  
-                          , function (er, uex) {
315  
-    if (er) return cb(er)
316  
-    userExclude = uex
317  
-    next()
318  
-  })
319  
-
320  
-  excludes.parseIgnoreFile( globalIgnore, null, dir
321  
-                          , function (er, gex) {
322  
-    if (er) return cb(er)
323  
-    globalExclude = gex
324  
-    next()
325  
-  })
326  
-
327  
-  function next () {
328  
-    if (!globalExclude || !userExclude) return
329  
-    var exList = [ defIgnore, confIgnore, globalExclude, userExclude ]
330  
-
331  
-    makeList_(dir, pkg, exList, dfc, function (er, files, cleanup) {
332  
-      if (er) return cb(er)
333  
-      var dirLen = dir.replace(/(\/|\\)$/, "").length + 1
334  
-      log.silly([dir, dirLen], "dir, dirLen")
335  
-      files = files.map(function (file) {
336  
-        return path.join(name, file.substr(dirLen))
337  
-      })
338  
-      return cb(null, files, cleanup)
339  
-    })
340  
-  }
341  
-}
342  
-
343  
-// Patterns ending in slashes will only match targets
344  
-// ending in slashes.  To implement this, add a / to
345  
-// the filename iff it lstats isDirectory()
346  
-function readDir (dir, pkg, dfc, cb) {
347  
-  fs.readdir(dir, function (er, files) {
348  
-    if (er) return cb(er)
349  
-    files = files.filter(function (f) {
350  
-      return f && f.charAt(0) !== "/" && f.indexOf("\0") === -1
351  
-    })
352  
-    asyncMap(files, function (file, cb) {
353  
-      fs.lstat(path.resolve(dir, file), function (er, st) {
354  
-        if (er) return cb(null, [])
355  
-        // if it's a directory, then tack "/" onto the name
356  
-        // so that it can match dir-only patterns in the
357  
-        // include/exclude logic later.
358  
-        if (st.isDirectory()) return cb(null, file + "/")
359  
-
360  
-        // if it's a symlink, then we need to do some more
361  
-        // complex stuff for GH-691
362  
-        if (st.isSymbolicLink()) return readSymlink(dir, file, pkg, dfc, cb)
363  
-
364  
-        // otherwise, just let it on through.
365  
-        return cb(null, file)
366  
-      })
367  
-    }, cb)
368  
-  })
369  
-}
370  
-
371  
-// just see where this link is pointing, and resolve relative paths.
372  
-function shallowReal (link, cb) {
373  
-  link = path.resolve(link)
374  
-  fs.readlink(link, function (er, t) {
375  
-    if (er) return cb(er)
376  
-    return cb(null, path.resolve(path.dirname(link), t), t)
377  
-  })
378  
-}
379  
-
380  
-function readSymlink (dir, file, pkg, dfc, cb) {
381  
-  var isNM = dfc
382  
-           && path.basename(dir) === "node_modules"
383  
-           && path.dirname(dir) === pkg.path
384  
-  // see if this thing is pointing outside of the package.
385  
-  // external symlinks are resolved for deps, ignored for other things.
386  
-  // internal symlinks are allowed through.
387  
-  var df = path.resolve(dir, file)
388  
-  shallowReal(df, function (er, r, target) {
389  
-    if (er) return cb(null, []) // wtf? exclude file.
390  
-    if (r.indexOf(dir) === 0) return cb(null, file) // internal
391  
-    if (!isNM) return cb(null, []) // external non-dep
392  
-    // now the fun stuff!
393  
-    fs.realpath(df, function (er, resolved) {
394  
-      if (er) return cb(null, []) // can't add it.
395  
-      readJson(path.resolve(resolved, "package.json"), function (er) {
396  
-        if (er) return cb(null, []) // not a package
397  
-        resolveLinkDep(dir, file, resolved, target, pkg, function (er, f, c) {
398  
-          cb(er, f, c)
399  
-        })
400  
-      })
401  
-    })
402  
-  })
403  
-}
404  
-
405  
-// put the link back the way it was.
406  
-function cleanupResolveLinkDep (cleanup, cb) {
407  
-  // cut it out of the list, so that cycles will be broken.
408  
-  if (!cleanup) return cb()
409  
-
410  
-  asyncMap(cleanup, function (d, cb) {
411  
-    rm(d[1], function (er) {
412  
-      if (er) return cb(er)
413  
-      fs.symlink(d[0], d[1], cb)
414  
-    })
415  
-  }, cb)
416  
-}
417  
-
418  
-function resolveLinkDep (dir, file, resolved, target, pkg, cb) {
419  
-  // we've already decided that this is a dep that will be bundled.
420  
-  // make sure the data reflects this.
421  
-  var bd = pkg.bundleDependencies || pkg.bundledDependencies || []
422  
-  delete pkg.bundledDependencies
423  
-  pkg.bundleDependencies = bd
424  
-  var f = path.resolve(dir, file)
425  
-    , cleanup = [[target, f, resolved]]
426  
-
427  
-  if (bd.indexOf(file) === -1) {
428  
-    // then we don't do this one.
429  
-    // just move the symlink out of the way.
430  
-    return rm(f, function (er) {
431  
-      cb(er, file, cleanup)
432  
-    })
433  
-  }
434  
-
435  
-  rm(f, function (er) {
436  
-    if (er) return cb(er)
437  
-    cache.add(resolved, function (er, data) {
438  
-      if (er) return cb(er)
439  
-      cache.unpack(data.name, data.version, f, function (er, data) {
440  
-        if (er) return cb(er)
441  
-        // now clear out the cache entry, since it's weird, probably.
442  
-        // pass the cleanup object along so that the thing getting the
443  
-        // list of files knows what to clean up afterwards.
444  
-        cache.clean([data._id], function (er) { cb(er, file, cleanup) })
445  
-      })
446  
-    })
447  
-  })
448  
-}
449  
-
450  
-// exList is a list of ignore lists.
451  
-// Each exList item is an array of patterns of files to ignore
452  
-//
453  
-function makeList_ (dir, pkg, exList, dfc, cb) {
454  
-  var files = null
455  
-    , cleanup = null
456  
-
457  
-  readDir(dir, pkg, dfc, function (er, f, c) {
458  
-    if (er) return cb(er)
459  
-    cleanup = c
460  
-    files = f.map(function (f) {
461  
-      // no nulls in paths!
462  
-      return f.split(/\0/)[0]
463  
-    }).filter(function (f) {
464  
-      // always remove all source control folders and
465  
-      // waf/vim/OSX garbage.  this is a firm requirement.
466  
-      return !( f === ".git/"
467  
-             || f === ".lock-wscript"
468  
-             || f.match(/^\.wafpickle-[0-9]+$/)
469  
-             || f === "CVS/"
470  
-             || f === ".svn/"
471  
-             || f === ".hg/"
472  
-             || f.match(/^\..*\.swp/)
473  
-             || f === ".DS_Store"
474  
-             || f.match(/^\._/)
475  
-             || f === "npm-debug.log"
476  
-             || f === ""
477  
-             || f.charAt(0) === "/"
478  
-              )
479  
-    })
480  
-
481  
-    // if (files.length > 0) files.push(".")
482  
-
483  
-    if (files.indexOf("package.json") !== -1 && dir !== pkg.path) {
484  
-      // a package.json file starts the whole exclude/include
485  
-      // logic all over.  Otherwise, a parent could break its
486  
-      // deps with its files list or .npmignore file.
487  
-      readJson(path.resolve(dir, "package.json"), function (er, data) {
488  
-        if (!er && typeof data === "object") {
489  
-          data.path = dir
490  
-          return makeList(dir, data, dfc, function (er, files) {
491  
-            // these need to be mounted onto the directory now.
492  
-            cb(er, files && files.map(function (f) {
493  
-              return path.resolve(path.dirname(dir), f)
494  
-            }))
495  
-          })
496  
-        }
497  
-        next()
498  
-      })
499  
-      //next()
500  
-    } else next()
501  
-
502  
-    // add a local ignore file, if found.
503  
-    if (files.indexOf(".npmignore") === -1
504  
-        && files.indexOf(".gitignore") === -1) next()
505  
-    else {
506  
-      excludes.addIgnoreFile( path.resolve(dir, ".npmignore")
507  
-                            , ".gitignore"
508  
-                            , exList
509  
-                            , dir
510  
-                            , function (er, list) {
511  
-        if (!er) exList = list
512  
-        next(er)
513  
-      })
514  
-    }
515  
-  })
516  
-
517  
-  var n = 2
518  
-    , errState = null
519  
-  function next (er) {
520  
-    if (errState) return
521  
-    if (er) return cb(errState = er, [], cleanup)
522  
-    if (-- n > 0) return
523  
-
524  
-    if (!pkg) return cb(new Error("No package.json file in "+dir))
525  
-    if (pkg.path === dir && pkg.files) {
526  
-      pkg.files = pkg.files.filter(function (f) {
527  
-        f = f.trim()
528  
-        return f && f.charAt(0) !== "#"
529  
-      })
530  
-      if (!pkg.files.length) pkg.files = null
531  
-    }
532  
-    if (pkg.path === dir && pkg.files) {
533  
-      // stuff on the files list MUST be there.
534  
-      // ignore everything, then include the stuff on the files list.
535  
-      var pkgFiles = ["*"].concat(pkg.files.map(function (f) {
536  
-        return "!" + f
537  
-      }))
538  
-      pkgFiles.dir = dir
539  
-      pkgFiles.packageFiles = true
540  
-      exList.push(pkgFiles)
541  
-    }
542  
-
543  
-    if (path.basename(dir) === "node_modules"
544  
-        && pkg.path === path.dirname(dir)
545  
-        // do fancy crap
546  
-        && dfc
547  
-        // not already part of a bundled dependency
548  
-        && (path.basename(path.dirname(pkg.path)) !== "node_modules"
549  
-          // unless it's the root
550  
-          || pkg.path === npm.prefix)) {
551  
-      log.verbose(dir, "doing fancy crap")
552  
-      files = filterNodeModules(files, pkg)
553  
-    } else {
554  
-      // If a directory is excluded, we still need to be
555  
-      // able to *include* a file within it, and have that override
556  
-      // the prior exclusion.
557  
-      //
558  
-      // This whole makeList thing probably needs to be rewritten
559  
-      files = files.filter(function (f) {
560  
-        return excludes.filter(dir, exList)(f) || f.slice(-1) === "/"
561  
-      })
562  
-    }
563  
-
564  
-
565  
-    asyncMap(files, function (file, cb) {
566  
-      // if this is a dir, then dive into it.
567  
-      // otherwise, don't.
568  
-      file = path.resolve(dir, file)
569  
-
570  
-      // in 0.6.0, fs.readdir can produce some really odd results.
571  
-      // XXX: remove this and make the engines hash exclude 0.6.0
572  
-      if (file.indexOf(dir) !== 0) {
573  
-        return cb(null, [])
574  
-      }
575  
-
576  
-      fs.lstat(file, function (er, st) {
577  
-        if (er) return cb(er)
578  
-        if (st.isDirectory()) {
579  
-          return makeList_(file, pkg, exList, dfc, cb)
580  
-        }
581  
-        return cb(null, file)
582  
-      })
583  
-    }, function (er, files, c) {
584  
-      if (c) cleanup = (cleanup || []).concat(c)
585  
-      if (files.length > 0) files.push(dir)
586  
-      return cb(er, files, cleanup)
587  
-    })
588  
-  }
589  
-}
590  
-
591  
-// only include node_modules folder that are:
592  
-// 1. not on the dependencies list or
593  
-// 2. on the "bundleDependencies" list.
594  
-function filterNodeModules (files, pkg) {
595  
-  var bd = pkg.bundleDependencies || pkg.bundledDependencies || []
596  
-    , deps = Object.keys(pkg.dependencies || {})
597  
-             .filter(function (key) { return !pkg.dependencies[key].extraneous })
598  
-             .concat(Object.keys(pkg.devDependencies || {}))
599  
-
600  
-  delete pkg.bundledDependencies
601  
-  pkg.bundleDependencies = bd
602  
-
603  
-  return files.filter(function (f) {
604  
-    f = f.replace(/\/$/, "")
605  
-    return f.charAt(0) !== "."
606  
-           && f.charAt(0) !== "_"
607  
-           && bd.indexOf(f) !== -1
608  
-  })
609  
-}

0 notes on commit 41daac7

Please sign in to comment.
Something went wrong with that request. Please try again.