From 00c63b7553885680edbd62da9edb86a4ddf10e92 Mon Sep 17 00:00:00 2001 From: Caolan McMahon Date: Thu, 9 Feb 2012 21:54:57 -0800 Subject: [PATCH] start re-implementing install command using tree builder --- lib/commands/install.js | 469 ++++++++++------------------------------ lib/commands/update.js | 1 + lib/packages.js | 4 + lib/tree.js | 51 ++++- test/test-lib-tree.js | 6 + 5 files changed, 166 insertions(+), 365 deletions(-) diff --git a/lib/commands/install.js b/lib/commands/install.js index b68a975..01d97cb 100644 --- a/lib/commands/install.js +++ b/lib/commands/install.js @@ -7,6 +7,7 @@ var semver = require('semver'), settings = require('../settings'), argParse = require('../args').parse, tar = require('../tar'), + tree = require('../tree'), async = require('async'), path = require('path'), fs = require('fs'); @@ -37,397 +38,157 @@ exports.usage = '' + ' --no-deps Don\'t fetch dependencies for the package'; -// set when run() is called -var OPTIONS = null; +/** + * Run function called when "kanso install" command is used + * + * @param {Object} settings - the values from .kansorc files + * @param {Array} args - command-line arguments + */ -// checked after queue.drain to see if it should exit cleanly -var ERRORS = false; - -// stores the version range requirements for each package -var pkg_ranges = {}; +exports.run = function (settings, args) { + var a = argParse(args, { + 'repository': {match: '--repository', value: true}, + 'force': {match: ['-f', '--force']}, + 'target_dir': {match: '--package-dir', value: true}, + 'no_deps': {match: '--no-deps'} + }); -// remembers if a package has already been removed when doing fetch -f -// otherwise, conflicting versions may overwrite each other without warning -var removed = {}; + var opt = a.options; + var pkg = a.positional[0] || '.'; -var processed = []; -var target_dir = './packages'; -var repos = []; + opt.repositories = settings.repositories; + if (a.options.repository) { + opt.repositories = [a.options.repository]; + // don't allow package dir .kansorc file to overwrite repositories + opt.fixed_repositories = true; + } + exports.install(pkg, opt, function (err) { + if (err) { + return logger.error(err); + } + logger.end(); + }); +}; -function isProcessed(name, version) { - for (var i = 0; i < processed.length; i++) { - if (processed[i].name === name && processed[i].version === version) { - return true; - } - } - return false; -} +/** + * Install a package from repository, file, directory or URL. + * + * @param {String} pkg - the package name, filename, directory or URL + * @param {Object} opt - options such as target_dir and repositories + * @param {Function} callback + */ -function getRanges(name) { - var ranges = []; - for (var k in (pkg_ranges[name] || {})) { - ranges.push(pkg_ranges[name][k]); +exports.install = function (pkg, opt, callback) { + var cache = {}; + if (/^https?:\/\//.test(pkg)) { + opt.target_dir = opt.target_dir || utils.abspath('packages'); + logger.info('installing from URL', pkg); + return exports.installURL(cache, pkg, opt, callback); } - return ranges; -} - + fs.stat(pkg, function (err, stats) { + if (err) { + // may not be a file + opt.target_dir = opt.target_dir || utils.abspath('packages'); + logger.info('installing from repositories', pkg); + return exports.installName(cache, pkg, opt, callback); + } + if (stats.isDirectory()) { + opt.target_dir = opt.target_dir || utils.abspath('packages', pkg); + logger.info('installing from directory', pkg); + return exports.installDir(cache, pkg, opt, callback); + } + else if (stats.isFile()) { + opt.target_dir = opt.target_dir || utils.abspath('packages'); + logger.info('installing from local file', pkg); + return exports.installFile(cache, pkg, opt, callback); + } + else { + return callback(new Error('Unknown install target: ' + pkg)); + } + }); +}; -function install(name, range, data, repo, parent, callback) { - pkg_ranges[name] = pkg_ranges[name] || {}; - pkg_ranges[name][parent] = range; - path.exists(target_dir + '/' + name, function (exists) { - if (exists) { - if (OPTIONS.force && !removed[name]) { - logger.info('removing', name); - utils.rm('-rf', target_dir + '/' + name, function (err) { +// check packages already in local target_dir +exports.dirSource = function (target_dir) { + return function (name, callback) { + var pdir = path.join(target_dir, name); + path.exists(pdir + '/kanso.json', function (exists) { + if (exists) { + settings.load(pdir, function (err, cfg) { if (err) { return callback(err); } - removed[name] = true; - process.nextTick(function () { - install(name, range, data, repo, parent, callback); - }); + var versions = {}; + versions[cfg.version] = cfg; + return callback(null, versions); }); } else { - settings.load(target_dir + '/' + name, function (err, cfg) { - if (err) { - return callback(err); - } - if (!versions.satisfiesAll(cfg.version, getRanges(name))) { - var range_data = ''; - for (var k in pkg_ranges[name]) { - range_data += '\n' + k + ' requires: ' + - pkg_ranges[name][k]; - } - range_data += '\n' + - 'Currently installed version: ' + cfg.version; - - return callback(new Error( - 'Conflicting version requirements for ' + name + - range_data - )); - } - logger.info('skipping', name + ' (already exists)'); - callback(null, cfg.version, cfg); - }); - } - return; - } - repository.fetch(name, range, repos, data, repo, - function (err, tfile, cdir, v, cfg, from_cache) { - if (err) { - return callback(err); - } - var path = target_dir + '/' + name; - logger.info( - 'installing', - name + '@' + v + (from_cache ? ' (cached)': '') - ); - utils.ensureDir(target_dir, function (err) { - if (err) { - return callback(err); - } - utils.cp('-r', cdir, path, function (err) { - if (err) { - return callback(err); - } - callback(null, v, cfg); - }); - }); + return callback(null, {}); } - ); - }); -} - -// tries to find a satisfying package locally, otherwise checks repositories -exports.resolve = function (parent, name, range, repos, callback) { - var pkgdir = path.join(target_dir, name); - path.exists(path.join(pkgdir, 'kanso.json'), function (exists) { - if (exists) { - settings.load(pkgdir, function (err, cfg) { - if (err) { - return callback(err); - } - if (semver.satisfies(cfg.version, range)) { - return callback(null, cfg.version, cfg, false); - } - else if (OPTIONS.force) { - repository.resolve( - name, range, repos, function (err, v, data, repo) { - if (err && err.notfound) { - return callback(new Error( - 'No package for ' + name + ' @ ' + range + - '\nAvailable versions: ' + cfg.version - )); - } - else { - callback.apply(this, arguments); - } - } - ); - } - else { - var range_data = ''; - for (var k in pkg_ranges[name]) { - range_data += '\n' + k + ' requires: ' + - pkg_ranges[name][k]; - } - range_data += '\n' + parent + ' requires: ' + range; - range_data += '\n' + - 'Currently installed version: ' + cfg.version; - - return callback(new Error( - 'Conflicting version requirements for ' + name + - range_data - )); - } - }); - } - else { - repository.resolve(name, range, repos, callback); - } - }); + }); + }; }; -function worker(task, callback) { - var name = task.name; - var range = task.range; - var parent = task.parent; - exports.resolve(parent, name, range, repos, function (err, v, data, repo) { - if (err) { - ERRORS = true; - logger.error(err); - return callback(err); - } - if (isProcessed(name, v)) { - return callback(); - } - processed.push({ - name: name, - version: v, - parent: parent +exports.repoSource = function (repositories) { + return function (name, callback) { + repository.availableVersions(name, repositories, function (err, vers) { + if (err) { + return callback(err); + } + var versions = {}; + for (var k in vers) { + // TODO: have tree module accept .cfg property instead? + versions[k] = vers[k].cfg; + } + return callback(null, versions); }); - var cfg = data; - if (repo) { - cfg = data.versions[v]; - } - if (cfg.dependencies && !OPTIONS['no-deps']) { - fetchDeps(cfg.dependencies, cfg.name); - } - if (repo) { - install(name, v, data, repo, parent, function (err) { - if (err) { - ERRORS = true; - logger.error(err); - return callback(err); - } - callback(); - }); - } - else { - logger.info('skipping', name + ' (already exists)'); - callback(); - } - }); -} + }; +}; -// the concurrency of fetch requests -var concurrency = 5; -var queue = async.queue(worker, concurrency); -function fetchDeps(deps, parent) { - Object.keys(deps).forEach(function (name) { - queue.push({name: name, range: deps[name], parent: parent}); - }); -} +/** + * Install a dependencies for a package directory. Reads the .kansorc and + * kanso.json files for that project and checks it's dependencies. + * + * @param {String} dir - the directory the install dependencies for + * @param {Object} opt - the options object + * @param {Function} callback + */ - -exports.installDir = function (_settings, dir) { - // install local package or project dir - if (OPTIONS['no-deps']) { - return logger.end(); +exports.installDir = function (cache, dir, opt, callback) { + if (opt.no_deps) { + return callback(); } - kansorc.extend(_settings, dir + '/.kansorc', function (err, _settings) { + kansorc.loadFile(dir + '/.kansorc', function (err, _settings) { if (err) { return logger.error(err); } + if (_settings.repositories && !opt.fixed_repositories) { + // overwrite repository list with package directory's list + opt.repositories = _settings.repositories; + } settings.load(dir, function (err, cfg) { if (err) { - ERRORS = true; - return logger.error(err); + return callback(err); } if (!cfg.dependencies) { logger.info('No dependencies specified'); - return logger.end(); - } - queue.drain = function () { - if (!ERRORS) { - return logger.end(); - } - }; - if (cfg.dependencies && !OPTIONS['no-deps']) { - fetchDeps(cfg.dependencies, cfg.name); - } - }); - }); -}; - -exports.installFile = function (_settings, filename) { - var tmp = repository.TMP_DIR + '/' + path.basename(filename); - var tmp_extracted = repository.TMP_DIR + '/package'; - - var callback = function (err) { - var args = arguments; - var that = this; - utils.rm('-rf', [tmp, tmp_extracted], function (err2) { - if (err2) { - ERRORS = true; - logger.error(err2); - } - else if (err) { - ERRORS = true; - logger.error(err); + return callback(); } - }); - }; - - async.series({ - tmpdir: async.apply(utils.ensureDir, repository.TMP_DIR), - dir: async.apply(utils.ensureDir, target_dir), - cp: function (cb) { - if (filename === tmp) { - // installing from a file in tmp already - return cb(); - } - utils.cp('-r', filename, tmp, cb); - }, - extract: async.apply(tar.extract, tmp), - cfg: async.apply(settings.load, tmp_extracted) - }, - function (err, results) { - if (err) { - return callback(err); - } - var v = results.cfg.version; - var name = results.cfg.name; - var tpath = target_dir + '/' + name; - path.exists(tpath, function (exists) { - if (exists) { - if (OPTIONS.force) { - logger.info('removing', name); - utils.rm('-rf', tpath, function (err) { - if (err) { - return callback(err); - } - process.nextTick(function () { - exports.installFile(_settings, filename); - }); - }); - } - else { - callback('"' + name + '" already exists in ' + target_dir); - } - return; - } - logger.info( - 'installing', - name + '@' + v + ' (from file)' - ); - utils.cp('-r', tmp_extracted, tpath, function (err) { + var sources = [ + exports.dirSource(opt.target_dir), + exports.repoSource(opt.repositories) + ]; + tree.build(cfg, sources, function (err, packages) { if (err) { return callback(err); } - // note that the --packge-dir won't have changed so deps will - // be installed in current packages dir alongside this one - exports.installDir(_settings, tpath); + console.log(packages); }); }); }); }; - -exports.installName = function (_settings, name) { - var version = 'latest'; - if (!name) { - ERRORS = true; - return logger.error('No package name specified'); - } - if (name.indexOf('@') !== -1) { - var parts = name.split('@'); - name = parts[0]; - version = parts.slice(1).join('@'); - } - var deps = {}; - deps[name] = version || null; - - queue.drain = function () { - if (!ERRORS) { - return logger.end(); - } - }; - fetchDeps(deps, null); -}; - - -exports.installURL = function (_settings, target) { - repository.download(target, function (err, filename) { - if (err) { - ERRORS = true; - return logger.error(err); - } - exports.installFile(_settings, filename); - }); -}; - - -exports.run = function (_settings, args) { - var a = argParse(args, { - 'repository': {match: '--repository', value: true}, - 'force': {match: ['-f', '--force']}, - 'package-dir': {match: '--package-dir', value: true}, - 'no-deps': {match: '--no-deps'} - }); - - // make sure package-dir is relative to current dir nor package paths - - OPTIONS = a.options; - - var pkg = a.positional[0] || '.'; - - repos = _settings.repositories; - if (a.options.repository) { - repos = [a.options.repository]; - } - - target_dir = a.options['package-dir']; - - if (/^https?:\/\//.test(pkg)) { - target_dir = target_dir || utils.abspath('packages'); - logger.info('installing from URL', pkg); - return exports.installURL(_settings, pkg); - } - fs.stat(pkg, function (err, stats) { - if (err) { - // may not be a file - target_dir = target_dir || utils.abspath('packages'); - logger.info('installing from repositories', pkg); - return exports.installName(_settings, pkg); - } - if (stats.isDirectory()) { - target_dir = target_dir || utils.abspath('packages', pkg); - logger.info('installing from directory', pkg); - return exports.installDir(_settings, pkg); - } - else if (stats.isFile()) { - target_dir = target_dir || utils.abspath('packages'); - logger.info('installing from local file', pkg); - return exports.installFile(_settings, pkg); - } - else { - throw new Error('Unknown install target: ' + pkg); - } - }); -}; diff --git a/lib/commands/update.js b/lib/commands/update.js index 29ed72c..e07e103 100644 --- a/lib/commands/update.js +++ b/lib/commands/update.js @@ -100,6 +100,7 @@ exports.getOutdated = function (repos, settings, dir, callback) { }); var outdated = []; logger.info('Checking for outdated packages...'); + async.forEachLimit(names, concurrency, function (k, cb) { console.log('Checking ' + k); var ranges = pcache[k].ranges; diff --git a/lib/packages.js b/lib/packages.js index 9dd215f..731ff5d 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -70,6 +70,10 @@ exports.load = function (name, paths, source, options, callback) { }, function (err, docs) { + if (err) { + return callback(err); + } + // combine all preprocessed design docs var doc = docs.reduce(function (a, b) { return exports.merge(a, b); diff --git a/lib/tree.js b/lib/tree.js index 40453dc..d7b4d32 100644 --- a/lib/tree.js +++ b/lib/tree.js @@ -4,6 +4,7 @@ */ var semver = require('semver'), + events = require('events'), async = require('async'), versions = require('./versions'), _ = require('underscore/underscore')._; @@ -61,6 +62,9 @@ exports.extend = function (pkg, sources, packages, callback) { packages[pkg.name].current_version = pkg.version; var dependencies = Object.keys(pkg.dependencies || {}); + if (!dependencies.length) { + return callback(null, packages); + } // TODO: split this into a separate exported function with a better name? function iterator(k, cb) { if (!packages[k]) { @@ -70,7 +74,7 @@ exports.extend = function (pkg, sources, packages, callback) { var curr = dep.current_version; dep.ranges[pkg.name] = pkg.dependencies[k]; - if (!semver.satisfies(curr, pkg.dependencies[k])) { + if (!curr || !versions.satisfiesAll(curr, Object.keys(dep.ranges))) { var available = Object.keys(dep.versions); var ranges = _.values(dep.ranges); var match = versions.maxSatisfying(available, ranges); @@ -79,23 +83,16 @@ exports.extend = function (pkg, sources, packages, callback) { dep.current_version = match; exports.extend(dep.versions[match], sources, packages, cb); } - else if (dep.sources.length) { - // TODO check sources - var fn = dep.sources.shift(); - fn(k, function (err, versions) { + else { + return exports.updateDep(k, dep, function (err) { if (err) { return cb(err); } - // keep existing versions, only add new ones - dep.versions = _.extend(versions, dep.versions); // re-run iterator with original args now there are // new versions available - iterator(k, cb); + return iterator(k, cb); }); } - else { - return cb(exports.dependencyError(k, dep)); - } } } async.forEach(dependencies, iterator, function (err) { @@ -104,6 +101,38 @@ exports.extend = function (pkg, sources, packages, callback) { }; +exports.updateDep = function (name, dep, callback) { + if (dep.update_in_progress) { + return dep.ev.once('update', callback); + } + else if (dep.sources.length) { + var fn = dep.sources.shift(); + if (!dep.ev) { + dep.ev = new events.EventEmitter(); + dep.ev.setMaxListeners(10000); + } + dep.update_in_progress = true; + + return fn(name, function (err, versions) { + if (err) { + return callback(err); + } + // keep existing versions, only add new ones + dep.versions = _.extend(versions, dep.versions); + + dep.update_in_progress = false; + dep.ev.emit('update'); + // re-run iterator with original args now there are + // new versions available + return callback(); + }); + } + else { + return callback(exports.dependencyError(name, dep)); + } +}; + + /** * Creates a new empty package object for adding to the version tree */ diff --git a/test/test-lib-tree.js b/test/test-lib-tree.js index 1ef3015..5d67d77 100644 --- a/test/test-lib-tree.js +++ b/test/test-lib-tree.js @@ -211,6 +211,7 @@ exports['build - fetch from sources'] = function (test) { } ]; tree.build(foo, sources, function (err, packages) { + delete packages['bar'].ev; test.same(packages, { 'foo': { versions: { @@ -239,6 +240,7 @@ exports['build - fetch from sources'] = function (test) { dependencies: {} } }, + update_in_progress: false, ranges: {'foo': '>= 0.0.2'}, sources: [], current_version: '0.0.2' @@ -294,6 +296,7 @@ exports['build - check multiple sources'] = function (test) { ]; tree.build(foo, sources, function (err, packages) { test.same(source_calls, ['one', 'two']); + delete packages['bar'].ev; test.same(packages, { 'foo': { versions: { @@ -324,6 +327,7 @@ exports['build - check multiple sources'] = function (test) { }, ranges: {'foo': '>= 0.0.2'}, sources: [], + update_in_progress: false, current_version: '0.0.2' } }); @@ -381,6 +385,7 @@ exports['build - check only as many sources as needed'] = function (test) { ]; tree.build(foo, sources, function (err, packages) { test.same(source_calls, ['one']); + delete packages['bar'].ev; test.same(packages, { 'foo': { versions: { @@ -411,6 +416,7 @@ exports['build - check only as many sources as needed'] = function (test) { }, ranges: {'foo': '>= 0.0.2'}, sources: [sources[1]], + update_in_progress: false, current_version: '0.0.2' } });