diff --git a/.travis.yml b/.travis.yml index abd71c4..5b75985 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,17 +12,13 @@ os: env: - NODE_VERSION="5" - - NODE_VERSION="4.2" + - NODE_VERSION="4" - NODE_VERSION="iojs-3" - NODE_VERSION="iojs-2" - NODE_VERSION="iojs-1" - NODE_VERSION="0.12" - NODE_VERSION="0.10" -#matrix: -# allow_failures: -# - os: osx - install: - rm -rf ~/.nvm - git clone https://github.com/creationix/nvm.git ~/.nvm diff --git a/bin/ynpm-cli.js b/bin/ynpm-cli.js index fe90425..2fdddb5 100644 --- a/bin/ynpm-cli.js +++ b/bin/ynpm-cli.js @@ -15,7 +15,9 @@ var yutils = require( '../lib/utils/yearn-utils' )( config ); var LOGGER = require( '../lib/utils/logger' ).getLOGGER( config.logger ); // Set version number -commander.version( version ); +commander + .version( version ) + .option( '-u, --unsafe', 'Run commands in unsafe mode.' ); // Link ynpm version to ynpm --version commander @@ -99,8 +101,12 @@ commander if( root_dir === undefined ) root_dir = process.cwd( ); - ynpm.commands.shrinkwrap( root_dir, {}, function( err, shrinkwrap ){ - fs.writeFileSync( path.join( root_dir, 'ynpm-shrinkwrap.json' ), JSON.stringify( shrinkwrap, null, '\t' ) ); + var unsafe = commander.unsafe; + + ynpm.commands.shrinkwrap( root_dir, {}, unsafe, function( err, shrinkwrap ){ + var dest = path.join( root_dir, 'ynpm-shrinkwrap.json' ); + fs.writeFileSync( dest, JSON.stringify( shrinkwrap, null, '\t' ) ); + console.log( 'Successfully generated ' + path.resolve( dest ) ); } ); } ); diff --git a/lib/utils/yearn-utils.js b/lib/utils/yearn-utils.js index 46ed6fb..9721f58 100644 --- a/lib/utils/yearn-utils.js +++ b/lib/utils/yearn-utils.js @@ -171,6 +171,55 @@ module.exports = function( config ){ return null; }; + yutils.findModuleLocationAsync = function( package_location, desired, callback ){ + + // Check if allowing legacy yearning + if( yutils.isLegacyYearning( desired ) || config.legacy ){ + yutils.LOGGER.warn( 'Assuming legacy yearning.' ); + + yutils.findLegacyModuleLocationAsync( path.dirname( package_location ), desired.module, function( module_location ){ + if( module_location !== null ) + return callback( null ); + + processModernYearning( ); + } ); + } else { + processModernYearning( ); + } + + function processModernYearning( ){ + // Hunt for satisfying module. + yutils.findModernModuleLocationAsync( + path.dirname( package_location ), + ( config.orgs[ desired.org ] !== undefined ? config.orgs[ desired.org ] : config.orgs[ '*' ].replace( /\*/g, desired.org ) ), + desired.module, + function( module_location ){ + fs.access( module_location, function( err ){ + if( err ){ + return callback( null ); + } + + // Look for "org/module_name/version" + fs.readdir( module_location, function( err, versions ){ + var available_versions = versions.filter( function( version ){ + // Filter out bad versions + return yutils.isValidSemVer( version ); + }).sort( function( a, b ){ return semver.rcompare( a, b, config.loose_semver ); } ); + + for( var index = 0; index < available_versions.length; ++index ){ + if( semver.satisfies( available_versions[ index ], desired.version, config.loose_semver ) ){ + return callback( path.resolve( path.join( module_location, available_versions[ index ] ) ) ); + } + } + + return callback( null ); + } ); + } ); + } + ); + } + }; + yutils.findModernModuleLocation = function( package_location, org_location, module ){ var root_location = path.resolve( package_location, org_location ); yutils.LOGGER.debug( 'Checking for module in ' + root_location ); @@ -181,6 +230,19 @@ module.exports = function( config ){ return null; }; + + yutils.findModernModuleLocationAsync = function( package_location, org_location, module, callback ){ + var root_location = path.resolve( package_location, org_location ); + yutils.LOGGER.debug( 'Checking for module in ' + root_location ); + + fs.access( path.join( root_location, module ), function( err ){ + if( err ){ + return callback( null ); + } + + callback( path.join( root_location, module ) ); + } ); + }; yutils.findLegacyModuleLocation = function( package_location, module ){ var root_location = path.resolve( package_location, './node_modules' ); diff --git a/lib/utils/ynpm-utils.js b/lib/utils/ynpm-utils.js index 908a51c..bab5823 100644 --- a/lib/utils/ynpm-utils.js +++ b/lib/utils/ynpm-utils.js @@ -40,9 +40,17 @@ module.exports = function( config, npm ){ } ); }; - nutils.findOrgs = function( desired_module, cwd ){ + nutils.findOrgs = function( desired_module, cwd, callback ){ var orgs = []; + var available_orgs = Object.keys( config.orgs ); + var count = available_orgs.length; + + function done( ){ + if( --count <= 0 ) + return callback( orgs ); + } + Object.keys( config.orgs ).forEach( function( org ){ // Check in wildcard orgs @@ -50,10 +58,20 @@ module.exports = function( config, npm ){ var root_path = path.resolve( config.orgs[ org ] ); root_path = root_path.substr( 0, root_path.indexOf( '*' ) ); - fs.readdirSync( root_path ).forEach( function( found_org ){ - if( fs.existsSync( path.join( config.orgs[ '*' ].replace( /\*/g, found_org ), desired_module ) ) ){ - orgs.push( found_org ); + fs.readdir( root_path, function( err, files ){ + var count = files.length; + function complete( ){ + if( --count <= 0 ) + return done( ); } + + files.forEach( function( found_org ){ + fs.access( path.join( config.orgs[ '*' ].replace( /\*/g, found_org ), desired_module ), function( err ){ + if( !err ) + orgs.push( found_org ); + return complete( ); + } ); + } ); } ); return; @@ -61,22 +79,27 @@ module.exports = function( config, npm ){ // Check in legacy locations if( org === '' && config.orgs[ org ] === './node_modules' ){ - if( fs.existsSync( path.resolve( cwd, path.join( config.orgs[ org ], desired_module ) ) ) ){ - orgs.push( org ); - } + fs.access( path.resolve( cwd, path.join( config.orgs[ org ], desired_module ) ), function ( err ){ + if( !err ) + orgs.push( org ); + done( ); + } ); + return; } // Check in direct org - if( fs.existsSync( path.join( config.orgs[ org ], desired_module ) ) ){ - orgs.push( org ); - } + fs.access( path.join( config.orgs[ org ], desired_module ), function( err ){ + if( !err ) + orgs.push( org ); + done( ); + } ); } ); - return orgs; + //return orgs; }; - nutils.findVersions = function( desired_org, desired_module, cwd ){ + nutils.findVersions = function( desired_org, desired_module, cwd, callback ){ var module_path; // Check in legacy locations @@ -90,18 +113,23 @@ module.exports = function( config, npm ){ module_path = path.join( config.orgs[ '*' ].replace( /\*/g, desired_org ), desired_module ); } - if( !fs.existsSync( module_path ) ) - return []; - - return fs.readdirSync( module_path ).filter( function( version ){ - return semver.valid( version, config.loose_semver ); - } ).sort( semver.compareLoose ); + return fs.readdir( module_path, function( err, files ){ + if( err ){ + return callback( [] ); + } + + callback( files.filter( function( version ){ + return semver.valid( version, config.loose_semver ); + } ).sort( semver.compareLoose ) ); + } ); }; - nutils.findMatchingVersions = function( desired_org, desired_module, desired_range , cwd ){ - return nutils.findVersions( desired_org, desired_module, cwd ).filter( function( version ){ - return semver.satisfies( version, desired_range, config.loose_semver ); - } ); + nutils.findMatchingVersions = function( desired_org, desired_module, desired_range, cwd, callback ){ + nutils.findVersions( desired_org, desired_module, cwd, function( versions ){ + callback( versions.filter( function( version ){ + return semver.satisfies( version, desired_range, config.loose_semver ); + } ) ); + } ); }; nutils.translateLegacyDependencyStructure = function( src_root, dest_primary_root, dest_secondary_root ){ diff --git a/lib/ynpm.js b/lib/ynpm.js index bc8baa6..0c802c5 100644 --- a/lib/ynpm.js +++ b/lib/ynpm.js @@ -1,10 +1,11 @@ var path = require( 'path' ); -var fs = require( 'fs' ); +var fs = require( 'fs-extra' ); var semver = require( 'semver' ); var npm = require( 'npm' ); var JSON5 = require( 'json5' ); var _ = require( 'lodash' ); +var async = require( 'async' ); module.exports = function( config, callback ){ @@ -65,37 +66,51 @@ module.exports = function( config, callback ){ } ); }; - ynpm.commands.list = function( desired, cwd, callback ){ - desired = yutils.extractYearningParts( desired ); + ynpm.commands.list = function( _desired, cwd, callback ){ + var desired = yutils.extractYearningParts( _desired ); - var orgs; if( desired.org !== undefined ){ - orgs = [ desired.org ]; + resolveVersions( [ desired.org ] ); } else { - orgs = nutils.findOrgs( desired.module, cwd ); + nutils.findOrgs( desired.module, cwd, resolveVersions ); } - - var list = []; - orgs.forEach( function( org ){ - var versions; - - if( desired.version === undefined ){ - versions = nutils.findVersions( org, desired.module, cwd ); - } else { - versions = nutils.findMatchingVersions( org, desired.module, desired.version, cwd ); - } - - versions.forEach( function( version ){ - list.push( yutils.constructYearningString( { org: ( org === '' ? undefined : org ), module: desired.module, version: version } ) ); - } ); - } ); - - callback( null, list ); + + function resolveVersions( orgs ){ + var list = []; + + var count = orgs.length; + function done( ){ + if( --count <= 0 ) + callback( null, list ); + } + + orgs.forEach( function( org ){ + + if( desired.version === undefined ){ + nutils.findVersions( org, desired.module, cwd, processVersions ); + } else { + nutils.findMatchingVersions( org, desired.module, desired.version, cwd, processVersions ); + } + + function processVersions( versions ){ + versions.forEach( function( version ){ + list.push( yutils.constructYearningString( { org: ( org === '' ? undefined : org ), module: desired.module, version: version } ) ); + } ); + done( ); + } + } ); + } }; - ynpm.commands.shrinkwrap = function( _cwd, _dependencies_seed, _callback ){ + ynpm.commands.shrinkwrap = function( _cwd, _dependencies_seed, unsafe, _callback ){ + + var queue = async.queue( function( task, callback ){ + nutils.findMatchingVersions( task.desired.org, task.desired.module, task.desired.version, task.cwd, function( versions ){ + callback( null, versions ); + } ); + }, 8 ); - function shrinkwrap_fn( cwd, dependencies_seed, callback ){ + function shrinkwrap_fn( cwd, dependencies_seed, dependency_chain, callback ){ var package_location = path.join( cwd, 'package.json' ); var shrinkwrap_location = path.join( cwd, 'ynpm-shrinkwrap.json' ); var pkg, local_shrinkwrap; @@ -130,13 +145,34 @@ module.exports = function( config, callback ){ desired.org = ( desired.org === undefined ? '' : desired.org ); desired.version = ( object_semver ? merged_dependencies[ dependency ].version : merged_dependencies[ dependency ] ); - desired.version = nutils.findMatchingVersions( desired.org, desired.module, desired.version, cwd )[ 0 ]; - - var dependency_root = yutils.findModuleLocation( package_location, desired ); - - shrinkwrap_fn( dependency_root, merged_dependencies[ dependency ].dependencies, function( error, dep_shrinkwrap ){ - shrinkwrap.dependencies[ dependency ] = dep_shrinkwrap; - return done(); + //nutils.findMatchingVersions( desired.org, desired.module, desired.version, cwd, function( versions ){ + queue.push( { desired: desired, cwd: cwd }, function( err, versions ){ + if( versions.length === 0 ){ + var message = 'Failed to find an acceptable version for ' + yutils.constructYearningString( desired ) + '.'; + + if( unsafe ){ + console.error( message ); + return done( ); + } else { + throw new Error( message ); + } + } + + desired.version = versions[ 0 ]; + var name = yutils.constructYearningString( desired ); + //console.log( 'Adding: ' + yutils.constructYearningString( desired ) ); + + if( dependency_chain.indexOf( name ) !== -1 ){ + shrinkwrap.dependencies[ dependency ] = desired.version; + return done( ); + } + + yutils.findModuleLocationAsync( package_location, desired, function( dependency_root ){ + shrinkwrap_fn( dependency_root, merged_dependencies[ dependency ].dependencies, dependency_chain.concat( name ), function( error, dep_shrinkwrap ){ + shrinkwrap.dependencies[ dependency ] = dep_shrinkwrap; + return done(); + } ); + } ); } ); } ); } @@ -164,7 +200,7 @@ module.exports = function( config, callback ){ } ); } - shrinkwrap_fn( _cwd, _dependencies_seed, function( err, shrinkwrap ){ + shrinkwrap_fn( _cwd, _dependencies_seed, [], function( err, shrinkwrap ){ if( typeof shrinkwrap === 'object' ) shrinkwrap.name = JSON5.parse( fs.readFileSync( path.join( _cwd, 'package.json' ) ) ).name; diff --git a/package.json b/package.json index dd013fc..222373a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "yearn", - "version": "0.8.0", + "version": "0.8.1", "description": "Override node's require mechanism.", "keywords": [ "require", @@ -34,10 +34,11 @@ "ynpm": "./bin/ynpm-cli.js" }, "dependencies": { + "async": "1.5.x", "commander": "2.9.x", "fs-extra": "0.26.x", "json5": "0.4.x", - "lodash": "4.2.x", + "lodash": "4.6.x", "npm": "2.14.x", "semver": "5.1.x", "temp": "0.8.x" @@ -47,8 +48,8 @@ }, "devDependencies": { "grunt": "^0.4.x", - "grunt-contrib-jshint": "^0.12.x", - "grunt-contrib-nodeunit": "^0.4.x", + "grunt-contrib-jshint": "^1.0.x", + "grunt-contrib-nodeunit": "^1.0.x", "grunt-coveralls": "^1.0.x", "istanbul": "^0.4.x" } diff --git a/tests/utils/ynpm-utils-find-tests.js b/tests/utils/ynpm-utils-find-tests.js index 0c71105..685842c 100644 --- a/tests/utils/ynpm-utils-find-tests.js +++ b/tests/utils/ynpm-utils-find-tests.js @@ -48,57 +48,45 @@ module.exports.setUp = function( callback ){ module.exports.findOrgsTests = { findOrgsNoneTest: function( unit ){ - unit.deepEqual( - ynpm_utils.findOrgs( 'missing', path.resolve( __dirname, '..' ) ), - [ ] - ); - - unit.done( ); + ynpm_utils.findOrgs( 'missing', path.resolve( __dirname, '..' ), function( orgs ){ + unit.deepEqual( orgs, [ ] ); + unit.done( ); + } ); }, findOrgsATest: function( unit ){ - unit.deepEqual( - ynpm_utils.findOrgs( 'A', path.resolve( __dirname, '..' ) ), - [ 'alphabet' ] - ); - - unit.done( ); + ynpm_utils.findOrgs( 'A', path.resolve( __dirname, '..' ), function( orgs ){ + unit.deepEqual( orgs, [ 'alphabet' ] ); + unit.done( ); + } ); }, findOrgsBTest: function( unit ){ - unit.deepEqual( - ynpm_utils.findOrgs( 'B', path.resolve( __dirname, '..' ) ), - [ 'alphabet' ] - ); - - unit.done( ); + ynpm_utils.findOrgs( 'B', path.resolve( __dirname, '..' ), function( orgs ){ + unit.deepEqual( orgs, [ 'alphabet' ] ); + unit.done( ); + } ); }, findOrgsJson5Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findOrgs( 'json5', path.resolve( __dirname, '..' ) ), - [ 'other' ] - ); - - unit.done( ); + ynpm_utils.findOrgs( 'json5', path.resolve( __dirname, '..' ), function( orgs ){ + unit.deepEqual( orgs, [ 'other' ] ); + unit.done( ); + } ); }, findOrgsTestModule1Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findOrgs( 'test-module-1', path.resolve( __dirname, '..' ) ), - [ '' ] - ); - - unit.done( ); + ynpm_utils.findOrgs( 'test-module-1', path.resolve( __dirname, '..' ), function( orgs ){ + unit.deepEqual( orgs, [ '' ] ); + unit.done( ); + } ); }, findOrgsTestModule3Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findOrgs( 'test-module-3', path.resolve( __dirname, '..' ) ), - [ '' ] - ); - - unit.done( ); + ynpm_utils.findOrgs( 'test-module-3', path.resolve( __dirname, '..' ), function( orgs ){ + unit.deepEqual( orgs, [ '' ] ); + unit.done( ); + } ); } }; @@ -106,113 +94,89 @@ module.exports.findOrgsTests = { module.exports.findVersionsTests = { findVersionsNoneTest: function( unit ){ - unit.deepEqual( - ynpm_utils.findVersions( 'alphabet', 'missing', path.resolve( __dirname, '..' ) ), - [ ] - ); - - unit.done( ); + ynpm_utils.findVersions( 'alphabet', 'missing', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ ] ); + unit.done( ); + } ); }, findVersionsATest: function( unit ){ - unit.deepEqual( - ynpm_utils.findVersions( 'alphabet', 'A', path.resolve( __dirname, '..' ) ), - [ '0.0.1', '0.0.2', '0.1.0' ] - ); - - unit.done( ); + ynpm_utils.findVersions( 'alphabet', 'A', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '0.0.1', '0.0.2', '0.1.0' ] ); + unit.done( ); + } ); }, findVersionsBTest: function( unit ){ - unit.deepEqual( - ynpm_utils.findVersions( 'alphabet', 'B', path.resolve( __dirname, '..' ) ), - [ '0.0.1', '0.0.2', '0.1.0' ] - ); - - unit.done( ); + ynpm_utils.findVersions( 'alphabet', 'B', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '0.0.1', '0.0.2', '0.1.0' ] ); + unit.done( ); + } ); }, findVersionsJson5Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findVersions( 'other', 'json5', path.resolve( __dirname, '..' ) ), - [ '0.0.1' ] - ); - - unit.done( ); + ynpm_utils.findVersions( 'other', 'json5', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '0.0.1' ] ); + unit.done( ); + } ); }, findVersionsTestModule1Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findVersions( '', 'test-module-1', path.resolve( __dirname, '..' ) ), - [ '1.0.0' ] - ); - - unit.done( ); + ynpm_utils.findVersions( '', 'test-module-1', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '1.0.0' ] ); + unit.done( ); + } ); }, findVersionsTestModule3Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findVersions( '', 'test-module-3', path.resolve( __dirname, '..' ) ), - [ '1.1.0', '2015.01.01-1' ] - ); - - unit.done( ); + ynpm_utils.findVersions( '', 'test-module-3', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '1.1.0', '2015.01.01-1' ] ); + unit.done( ); + } ); } }; module.exports.findMatchingVersionsTests = { findMatchingVersionsNoneTest: function( unit ){ - unit.deepEqual( - ynpm_utils.findMatchingVersions( 'alphabet', 'missing', '*', path.resolve( __dirname, '..' ) ), - [ ] - ); - - unit.done( ); + ynpm_utils.findMatchingVersions( 'alphabet', 'missing', '*', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ ] ); + unit.done( ); + } ); }, findMatchingVersionsATest: function( unit ){ - unit.deepEqual( - ynpm_utils.findMatchingVersions( 'alphabet', 'A', '*', path.resolve( __dirname, '..' ) ), - [ '0.0.1', '0.0.2', '0.1.0' ] - ); - - unit.done( ); + ynpm_utils.findMatchingVersions( 'alphabet', 'A', '*', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '0.0.1', '0.0.2', '0.1.0' ] ); + unit.done( ); + } ); }, findMatchingVersionsBTest: function( unit ){ - unit.deepEqual( - ynpm_utils.findMatchingVersions( 'alphabet', 'B', '>=0.1.0', path.resolve( __dirname, '..' ) ), - [ '0.1.0' ] - ); - - unit.done( ); + ynpm_utils.findMatchingVersions( 'alphabet', 'B', '>=0.1.0', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '0.1.0' ] ); + unit.done( ); + } ); }, findMatchingVersionsJson5Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findMatchingVersions( 'other', 'json5', '0.1.x', path.resolve( __dirname, '..' ) ), - [ ] - ); - - unit.done( ); + ynpm_utils.findMatchingVersions( 'other', 'json5', '0.1.x', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ ] ); + unit.done( ); + } ); }, findMatchingVersionsTestModule1Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findMatchingVersions( '', 'test-module-1', '*', path.resolve( __dirname, '..' ) ), - [ '1.0.0' ] - ); - - unit.done( ); + ynpm_utils.findMatchingVersions( '', 'test-module-1', '*', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '1.0.0' ] ); + unit.done( ); + } ); }, findMatchingVersionsTestModule3Test: function( unit ){ - unit.deepEqual( - ynpm_utils.findMatchingVersions( '', 'test-module-3', '*', path.resolve( __dirname, '..' ) ), - [ '1.1.0' /*, '2015.01.01-1' // Doesn't find this because of semver validation of prerelease tags */ ] - ); - - unit.done( ); + ynpm_utils.findMatchingVersions( '', 'test-module-3', '*', path.resolve( __dirname, '..' ), function( versions ){ + unit.deepEqual( versions, [ '1.1.0' /*, '2015.01.01-1' // Doesn't find this because of semver validation of prerelease tags */ ] ); + unit.done( ); + } ); } }; \ No newline at end of file diff --git a/tests/ynpm-shrinkwrap-tests.js b/tests/ynpm-shrinkwrap-tests.js index e7b46ee..8b345d7 100644 --- a/tests/ynpm-shrinkwrap-tests.js +++ b/tests/ynpm-shrinkwrap-tests.js @@ -44,7 +44,7 @@ module.exports.shrinkwrapCommandTests = { }, function( err, ynpm ){ test.strictEqual( err, null, 'No errors on ynpm initialization' ); - ynpm.commands.shrinkwrap( path.resolve( __dirname, './test-orgs/alphabet/D/0.1.0' ) , {}, function( err, shrinkwrap ){ + ynpm.commands.shrinkwrap( path.resolve( __dirname, './test-orgs/alphabet/D/0.1.0' ) , {}, false, function( err, shrinkwrap ){ test.strictEqual( err, null, 'No errors in shrinkwrap command.' ); test.deepEqual( shrinkwrap, { 'version': '0.1.0',