Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

refactor to be testable, and to allow multiple resolve strategies

  • Loading branch information...
commit e63ececa9f430686de63da29b6f308c5614b20ee 1 parent 281c24f
@dominictarr authored
Showing with 226 additions and 226 deletions.
  1. +0 −3  clean.js
  2. +22 −188 index.js
  3. +147 −0 resolve-tree.js
  4. +57 −35 resolve.js
View
3  clean.js
@@ -4,13 +4,10 @@ module.exports = function clean (pkg) {
var deps = pkg.dependencies
var _deps = pkg.tree || {}
-// var shasum = pkg.shasum
for(var k in pkg)
if(!~fields.indexOf(k))
delete pkg[k]
-// pkg.shasum = shasum
-
for(var k in _deps) {
_deps[k].from = deps[k]
clean(_deps[k])
View
210 index.js
@@ -1,198 +1,31 @@
var path = require('path')
-var pull = require('pull-stream')
-var pt = require('pull-traverse')
-var inspect = require('util').inspect
-var semver = require('semver')
-var cat = require('pull-cat')
-var urlResolve = require('npmd-git-resolve')
-var clean = require('./clean')
var ls = require('npmd-tree').ls
-var paramap = require('pull-paramap')
-var deps = require('get-deps')
-//experimenting with different installation resolve
-//algs. the idea is to traverse the tree locally,
-//figure out what is needed, and then install from
-//cache if possible. Else, get the rest in just one
-//bundle. (the registry can stream them into one
-//multipart or a tarball or something)
-
-//I've implemented the basic algorithm that npm uses,
-//and a greedy algorithm. the greedy algorithm
-//would every module into $PWD/node_modules
-//and only creates new node_modules directories
-//when the version range specified is not available.
-
-//testing with the trees for large projects, (such as npm and browserify)
-//this may require 10-30% fewer installs
-
-//opening the database, and running this the first time is pretty slow
-//like 2 seconds to resolve browserify (50 modules)
-//but when the cache is warm it's only 50 ms!
-
-//however, I've recently moved on to a technique
-//that simlinks every module.
-//this allows exact control of the code in your dep tree
-//and also, it's really really fast.
-
-//see npmd-leaves and npmd-link
-
-var resolvePackage = require('./resolve')
-
-
-function check(pkg, name, range) {
- if(!pkg) return false
- if(pkg.tree[name] && semver.satisfies(pkg.tree[name].version, range))
- return true
- return check(pkg.parent, name, range)
-}
-
-function fixModule (module) {
- if('string' === typeof module) {
- var parts = module.split('@')
- return {name: parts.shift(), version: parts.shift() || '*'}
- }
- return module
-}
-
-function resolveTree (db, module, opts, cb) {
-
- module = fixModule(module)
-
- var filter = opts.filter || function (pkg, root) {
- if(!pkg) return
- pkg.parent.tree[pkg.name] = pkg
- }
-
- if (! module) return cb(null, {})
-
- resolvePackage(db, module.name, module.version, function (err, pkg) {
- if(err) return cb(err)
-
- var root = pkg
-
- if(opts.available) {
- root.parent = {tree: opts.available}
- }
-
- pull(cat([
- pull.values([pkg]),
- pt.depthFirst(pkg, function (pkg) {
- var deps = pkg.dependencies || {}
- pkg.tree = {}
- return pull(
- pull.values(Object.keys(deps)),
- //this could be parallel,
- //but it's not the bottle neck.
- paramap(function (name, cb) {
- //check if there is already a module that resolves this...
-
- //filter out versions that we already have.
- //if(opts.check !== false && check(pkg, name, deps[name]))
- // return cb()
-
- resolvePackage(db, name, deps[name], function (err, _pkg) {
- if(err) {
- err.message = 'package:' + pkg.name + '@' + pkg.version + ' could not resolve ' + name + '@'+deps[name] + '\n' +
- err.message
- }
- cb(err, _pkg)
- })
- }),
- pull.filter(),
- pull.through(function (_pkg) {
- _pkg.parent = pkg
- filter(_pkg, root)
- })
- )
- })
- ]),
- pull.drain(null, function (err) {
- cb(err, clean(pkg))
- }))
- })
-
-}
-
-function resolveTreeGreedy (db, module, opts, cb) {
- if(!cb) cb = opts, opts = null
-
- opts = opts || {}
- opts.filter = function (pkg, root) {
- if(!pkg) return
- //install non-conflicting modules as low in the tree as possible.
- //hmm, is this wrong?
- //hmm, the only way a module is not on the root is if it's
- //conflicting with one that is already there.
- //so, what if this module is a child of a conflicting module
- //aha! we have already checked the path to the root,
- //and this item would be filtered if it wasn't clear.
- if(!root.tree[pkg.name]) {
- root.tree[pkg.name] = pkg
- pkg.parent = root
- }
- else {
- pkg.parent.tree[pkg.name] = pkg
- }
- return pkg
- }
-
- resolveTree(db, module, opts, cb)
-}
-
-var resolve = exports = module.exports =
-function (db, module, opts, cb) {
- if(!cb)
- cb = opts, opts = {}
-
- function resolve(module, cb) {
- if(opts && opts.greedy)
- resolveTreeGreedy(db, module, opts, cb)
- else
- resolveTree(db, module, opts, cb)
- }
-
- if(Array.isArray(module) && module.length > 1) {
-
- var n = module.length, a = {}
-
- module.forEach(function (m) {
- resolve(m, next)
- })
-
- function next (err, tree) {
- if(err) return n = 0, cb(err)
- a[tree.name] = tree
- if(--n) return
- cb(null, a)
- }
-
- } else {
- if(Array.isArray(module))
- module = module.shift()
-
- resolve(module, cb)
- }
-
-}
-
-exports.resolveTree = resolveTree
-exports.resolveTreeGreedy = resolveTreeGreedy
+var createResolvePackage = require('./resolve')
+var createResolve = require('./resolve-tree')
exports.db = function (db, config) {
+ var resolvePackage = createResolvePackage(db.sublevel('ver'), config)
+ var resolve = createResolve(resolvePackage)
db.methods.resolve = {type: 'async'}
db.resolve = function (module, opts, cb) {
if(!cb) cb = opts, opts = {}
- if(opts.hash)
- opts.greedy = false
+ if(opts.hash) opts.greedy = false
resolve(
- db.sublevel('ver'), module,
+ module,
opts,
cb
)
}
}
+function merge (a, b) {
+ for(var k in b)
+ if(!a[k])
+ a[k] = b[k]
+ return a
+}
+
exports.cli = function (db) {
db.commands.push(function (db, config, cb) {
var args = config._.slice()
@@ -204,14 +37,15 @@ exports.cli = function (db) {
ls(function (err, tree) {
if(err) return cb(err)
- db.resolve(args, {
- greedy: config.greedy,
- available: tree
- },
- function (err, tree) {
- if(err) return cb(err)
- console.log(JSON.stringify(tree, null, 2))
- cb(null, tree)
+ db.resolve(args,
+ merge({
+ greedy: config.greedy,
+ available: tree
+ }, config),
+ function (err, tree) {
+ if(err) return cb(err)
+ console.log(JSON.stringify(tree, null, 2))
+ cb(null, tree)
})
})
return true
View
147 resolve-tree.js
@@ -0,0 +1,147 @@
+var pull = require('pull-stream')
+var pt = require('pull-traverse')
+var semver = require('semver')
+var cat = require('pull-cat')
+var clean = require('./clean')
+var paramap = require('pull-paramap')
+
+module.exports = createResolve
+
+function check(pkg, name, range) {
+ if(!pkg) return false
+ if(pkg.tree[name] && semver.satisfies(pkg.tree[name].version, range, true))
+ return true
+ return check(pkg.parent, name, range)
+}
+
+function fixModule (module) {
+ if('string' === typeof module) {
+ var parts = module.split('@')
+ return {name: parts.shift(), version: parts.shift() || '*'}
+ }
+ return module
+}
+
+function niceError(err, parent) {
+ if(!err) return
+ err.message = 'package:'
+ + pkg.name + '@' + pkg.version
+ + ' could not resolve '
+ + name + '@' + deps[name]
+ + '\n' + err.message
+
+ return err
+}
+
+function createResolve (resolvePackage) {
+
+ function resolveTree (module, opts, cb) {
+
+ module = fixModule(module)
+
+ var filter = opts.filter || function (pkg, root) {
+ if(!pkg) return
+ pkg.parent.tree[pkg.name] = pkg
+ }
+
+ if (! module) return cb(null, {})
+
+ resolvePackage(module.name, module.version, opts, function (err, pkg) {
+ if(err) return cb(err)
+
+ var root = pkg
+
+ if(opts.available) {
+ root.parent = {tree: opts.available}
+ }
+
+ pull(cat([
+ pull.values([pkg]),
+ pt.depthFirst(pkg, function (pkg) {
+ var deps = pkg.dependencies || {}
+ pkg.tree = {}
+ return pull(
+ pull.values(Object.keys(deps)),
+ //this could be parallel,
+ //but it's not the bottle neck.
+ paramap(function (name, cb) {
+ //check if there is already a module that resolves this...
+
+ //filter out versions that we already have.
+ //if(opts.check !== false && check(pkg, name, deps[name]))
+ // return cb()
+
+ resolvePackage(name, deps[name], opts, function (err, _pkg) {
+ cb(niceError(err, pkg), _pkg)
+ })
+ }),
+ pull.filter(),
+ pull.through(function (_pkg) {
+ _pkg.parent = pkg
+ filter(_pkg, root)
+ })
+ )
+ })
+ ]),
+ pull.drain(null, function (err) {
+ cb(err, clean(pkg))
+ }))
+ })
+
+ }
+
+ //install non-conflicting modules as low in the tree as possible.
+ function resolveTreeGreedy (module, opts, cb) {
+ if(!cb) cb = opts, opts = null
+
+ opts = opts || {}
+ opts.filter = function (pkg, root) {
+ if(!pkg) return
+ if(!root.tree[pkg.name]) {
+ root.tree[pkg.name] = pkg
+ pkg.parent = root
+ }
+ else {
+ pkg.parent.tree[pkg.name] = pkg
+ }
+ return pkg
+ }
+
+ resolveTree(resolvePackage, module, opts, cb)
+ }
+
+ return function (module, opts, cb) {
+ if(!cb)
+ cb = opts, opts = {}
+
+ function resolve(module, cb) {
+ if(opts && opts.greedy)
+ resolveTreeGreedy(module, opts, cb)
+ else
+ resolveTree(module, opts, cb)
+ }
+
+ if(Array.isArray(module) && module.length > 1) {
+
+ var n = module.length, a = {}
+
+ module.forEach(function (m) {
+ resolve(m, next)
+ })
+
+ function next (err, tree) {
+ if(err) return n = 0, cb(err)
+ a[tree.name] = tree
+ if(--n) return
+ cb(null, a)
+ }
+
+ } else {
+ if(Array.isArray(module))
+ module = module.shift()
+
+ resolve(module, cb)
+ }
+ }
+}
+
View
92 resolve.js
@@ -3,6 +3,7 @@ var fs = require('fs')
var path = require('path')
var range = require('padded-semver').range
var peek = require('level-peek')
+var urlResolve = require('npmd-git-resolve')
function all (db, module, cb) {
var found = {}
@@ -16,12 +17,12 @@ function all (db, module, cb) {
.on('error', cb)
}
-module.exports = resolvePackage
-
-function resolvePackage (db, module, vrange, opts, cb) {
- if(!cb) cb = opts, opts = {}
- if(!cb) cb = vrange, vrange = '*'
+function remoteResolve (module, vrange, opts, cb) {
+ //if vrange is a http or git
+ //download the bundle
+ //note - you can download git via an archive...
+ //then unpack to get package.json
if(/^(git|http|\w+\/)/.test(vrange)) {
console.error('GET', vrange)
return urlResolve(vrange, opts, function (err, pkg) {
@@ -32,40 +33,51 @@ function resolvePackage (db, module, vrange, opts, cb) {
cb(err, pkg)
})
}
- //if vrange is a http or git
- //download the bundle
- //note - you can download git via an archive...
- //then unpack to get package.json
- //
+ else cb()
+}
+
- if(vrange == 'latest')
- vrange = '*'
+function offlineResolve (module, vrange, opts, cb) {
+ fs.readdir(path.join(opts.cache, module), function (err, list) {
+ if(err) return cb(err)
+ list = list.filter(function (e) {
+ return semver.valid(e, true)
+ })
+ var ver = semver.maxSatisfying(list, vrange, true)
+ if(!ver)
+ return cb(new Error('no module satified version:' + vrange))
+ fs.readFile(
+ path.join(opts.cache, module, ver, 'package', 'package.json'),
+ 'utf-8',
+ function (err, json) {
+ if(err) return cb(err)
+ try { json = JSON.parse(json) } catch (err) { return cb(err) }
+ cb(null, json)
+ })
+ })
+}
- if(opts.correct === true) {
+module.exports = function (db, config) {
+
+ function correctResolve (module, vrange, opts, cb) {
all(db, module, function (err, versions) {
if(err) return cb(err)
var ver = semver.maxSatisfying(Object.keys(versions), vrange, true)
if(!ver)
return cb(new Error('no module satified version:' + vrange))
- if(opts.offline)
- return fs.readdir(path.join(process.env.HOME, '.npm', module), function (err, list) {
- var ver = semver.maxSatisfying(list, vrange, true)
- if(!ver)
- return cb(new Error('no module satified version:' + vrange))
- fs.readFile(path.join(process.env.HOME, '.npm', module, 'package', 'package.json'), 'utf-8',
- function (err, json) {
- if(err) return cb(err)
- try { json = JSON.parse(json) } catch (err) { return cb(err) }
- cb(null, json)
- })
- })
-
- cb(null, versions[ver])
+ cb(null, versions[ver])
})
+ }
+
+ function peekResolve (module, vrange, opts, cb) {
+ if(!cb) cb = opts, opts = {}
+ if(!cb) cb = vrange, vrange = '*'
+
+ if(vrange == 'latest')
+ vrange = '*'
- } else {
var r = range(vrange || '*')
r = {
@@ -74,15 +86,25 @@ function resolvePackage (db, module, vrange, opts, cb) {
}
peek.last(db, r, function (err, key, pkg) {
- if(err) {
- console.log(r, vrange)
- return cb(err)
- }
- if(!semver.satisfies(pkg.version, vrange || '*', true)) {
+ if(err) return cb(err)
+
+ if(!semver.satisfies(pkg.version, vrange || '*', true))
return cb(new Error(module+'@'+pkg.version +'><'+ vrange))
- }
+
cb(null, pkg)
})
}
-}
+ return function (module, vrange, opts, cb) {
+ if(!cb) throw new Error('expects cb')
+ remoteResolve(module, vrange, opts, function (err, pkg) {
+ if(pkg || err) return cb(err, pkg)
+ if(opts.offline) return offlineResolve(module, vrange, opts, cb)
+ if(opts.correct) return correctResolve(module, vrange, opts, cb)
+ peekResolve(module, vrange, opts, function (err, pkg) {
+ if(pkg) cb(null, pkg)
+ else correctResolve(module, vrange, opts, cb)
+ })
+ })
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.