Skip to content

Commit

Permalink
Fixed bug in ynpm shrinkwrap where large modules would infinitely loo…
Browse files Browse the repository at this point in the history 
…p and crash the system.
  • Loading branch information
@doctorrustynelson
doctorrustynelson committed Mar 7, 2016
1 parent 1e43c4f commit 918eabb
Show file tree
Hide file tree
Showing 8 changed files with 269 additions and 176 deletions.
6 changes: 1 addition & 5 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,13 @@ os:

env:
- NODE_VERSION="5"
- NODE_VERSION="4.2"
- NODE_VERSION="4"
- NODE_VERSION="iojs-3"
- NODE_VERSION="iojs-2"
- NODE_VERSION="iojs-1"
- NODE_VERSION="0.12"
- NODE_VERSION="0.10"

#matrix:
# allow_failures:
# - os: osx

install:
- rm -rf ~/.nvm
- git clone https://github.com/creationix/nvm.git ~/.nvm
Expand Down
12 changes: 9 additions & 3 deletions bin/ynpm-cli.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ var yutils = require( '../lib/utils/yearn-utils' )( config );
var LOGGER = require( '../lib/utils/logger' ).getLOGGER( config.logger );

// Set version number
commander.version( version );
commander
.version( version )
.option( '-u, --unsafe', 'Run commands in unsafe mode.' );

// Link ynpm version to ynpm --version
commander
Expand Down Expand Up @@ -99,8 +101,12 @@ commander
if( root_dir === undefined )
root_dir = process.cwd( );

ynpm.commands.shrinkwrap( root_dir, {}, function( err, shrinkwrap ){
fs.writeFileSync( path.join( root_dir, 'ynpm-shrinkwrap.json' ), JSON.stringify( shrinkwrap, null, '\t' ) );
var unsafe = commander.unsafe;

ynpm.commands.shrinkwrap( root_dir, {}, unsafe, function( err, shrinkwrap ){
var dest = path.join( root_dir, 'ynpm-shrinkwrap.json' );
fs.writeFileSync( dest, JSON.stringify( shrinkwrap, null, '\t' ) );
console.log( 'Successfully generated ' + path.resolve( dest ) );
} );
} );

Expand Down
62 changes: 62 additions & 0 deletions lib/utils/yearn-utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,55 @@ module.exports = function( config ){
return null;
};

yutils.findModuleLocationAsync = function( package_location, desired, callback ){

// Check if allowing legacy yearning
if( yutils.isLegacyYearning( desired ) || config.legacy ){
yutils.LOGGER.warn( 'Assuming legacy yearning.' );

yutils.findLegacyModuleLocationAsync( path.dirname( package_location ), desired.module, function( module_location ){
if( module_location !== null )
return callback( null );

processModernYearning( );
} );
} else {
processModernYearning( );
}

function processModernYearning( ){
// Hunt for satisfying module.
yutils.findModernModuleLocationAsync(
path.dirname( package_location ),
( config.orgs[ desired.org ] !== undefined ? config.orgs[ desired.org ] : config.orgs[ '*' ].replace( /\*/g, desired.org ) ),
desired.module,
function( module_location ){
fs.access( module_location, function( err ){
if( err ){
return callback( null );
}

// Look for "org/module_name/version"
fs.readdir( module_location, function( err, versions ){
var available_versions = versions.filter( function( version ){
// Filter out bad versions
return yutils.isValidSemVer( version );
}).sort( function( a, b ){ return semver.rcompare( a, b, config.loose_semver ); } );

for( var index = 0; index < available_versions.length; ++index ){
if( semver.satisfies( available_versions[ index ], desired.version, config.loose_semver ) ){
return callback( path.resolve( path.join( module_location, available_versions[ index ] ) ) );
}
}

return callback( null );
} );
} );
}
);
}
};

yutils.findModernModuleLocation = function( package_location, org_location, module ){
var root_location = path.resolve( package_location, org_location );
yutils.LOGGER.debug( 'Checking for module in ' + root_location );
Expand All @@ -181,6 +230,19 @@ module.exports = function( config ){

return null;
};

yutils.findModernModuleLocationAsync = function( package_location, org_location, module, callback ){
var root_location = path.resolve( package_location, org_location );
yutils.LOGGER.debug( 'Checking for module in ' + root_location );

fs.access( path.join( root_location, module ), function( err ){
if( err ){
return callback( null );
}

callback( path.join( root_location, module ) );
} );
};

yutils.findLegacyModuleLocation = function( package_location, module ){
var root_location = path.resolve( package_location, './node_modules' );
Expand Down
72 changes: 50 additions & 22 deletions lib/utils/ynpm-utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,43 +40,66 @@ module.exports = function( config, npm ){
} );
};

nutils.findOrgs = function( desired_module, cwd ){
nutils.findOrgs = function( desired_module, cwd, callback ){
var orgs = [];

var available_orgs = Object.keys( config.orgs );
var count = available_orgs.length;

function done( ){
if( --count <= 0 )
return callback( orgs );
}

Object.keys( config.orgs ).forEach( function( org ){

// Check in wildcard orgs
if( org === '*' ){
var root_path = path.resolve( config.orgs[ org ] );
root_path = root_path.substr( 0, root_path.indexOf( '*' ) );

fs.readdirSync( root_path ).forEach( function( found_org ){
if( fs.existsSync( path.join( config.orgs[ '*' ].replace( /\*/g, found_org ), desired_module ) ) ){
orgs.push( found_org );
fs.readdir( root_path, function( err, files ){
var count = files.length;
function complete( ){
if( --count <= 0 )
return done( );
}

files.forEach( function( found_org ){
fs.access( path.join( config.orgs[ '*' ].replace( /\*/g, found_org ), desired_module ), function( err ){
if( !err )
orgs.push( found_org );
return complete( );
} );
} );
} );

return;
}

// Check in legacy locations
if( org === '' && config.orgs[ org ] === './node_modules' ){
if( fs.existsSync( path.resolve( cwd, path.join( config.orgs[ org ], desired_module ) ) ) ){
orgs.push( org );
}
fs.access( path.resolve( cwd, path.join( config.orgs[ org ], desired_module ) ), function ( err ){
if( !err )
orgs.push( org );
done( );
} );

return;
}

// Check in direct org
if( fs.existsSync( path.join( config.orgs[ org ], desired_module ) ) ){
orgs.push( org );
}
fs.access( path.join( config.orgs[ org ], desired_module ), function( err ){
if( !err )
orgs.push( org );
done( );
} );
} );

return orgs;
//return orgs;
};

nutils.findVersions = function( desired_org, desired_module, cwd ){
nutils.findVersions = function( desired_org, desired_module, cwd, callback ){
var module_path;

// Check in legacy locations
Expand All @@ -90,18 +113,23 @@ module.exports = function( config, npm ){
module_path = path.join( config.orgs[ '*' ].replace( /\*/g, desired_org ), desired_module );
}

if( !fs.existsSync( module_path ) )
return [];

return fs.readdirSync( module_path ).filter( function( version ){
return semver.valid( version, config.loose_semver );
} ).sort( semver.compareLoose );
return fs.readdir( module_path, function( err, files ){
if( err ){
return callback( [] );
}

callback( files.filter( function( version ){
return semver.valid( version, config.loose_semver );
} ).sort( semver.compareLoose ) );
} );
};

nutils.findMatchingVersions = function( desired_org, desired_module, desired_range , cwd ){
return nutils.findVersions( desired_org, desired_module, cwd ).filter( function( version ){
return semver.satisfies( version, desired_range, config.loose_semver );
} );
nutils.findMatchingVersions = function( desired_org, desired_module, desired_range, cwd, callback ){
nutils.findVersions( desired_org, desired_module, cwd, function( versions ){
callback( versions.filter( function( version ){
return semver.satisfies( version, desired_range, config.loose_semver );
} ) );
} );
};

nutils.translateLegacyDependencyStructure = function( src_root, dest_primary_root, dest_secondary_root ){
Expand Down
102 changes: 69 additions & 33 deletions lib/ynpm.js
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@

var path = require( 'path' );
var fs = require( 'fs' );
var fs = require( 'fs-extra' );
var semver = require( 'semver' );
var npm = require( 'npm' );
var JSON5 = require( 'json5' );
var _ = require( 'lodash' );
var async = require( 'async' );

module.exports = function( config, callback ){

Expand Down Expand Up @@ -65,37 +66,51 @@ module.exports = function( config, callback ){
} );
};

ynpm.commands.list = function( desired, cwd, callback ){
desired = yutils.extractYearningParts( desired );
ynpm.commands.list = function( _desired, cwd, callback ){
var desired = yutils.extractYearningParts( _desired );

var orgs;
if( desired.org !== undefined ){
orgs = [ desired.org ];
resolveVersions( [ desired.org ] );
} else {
orgs = nutils.findOrgs( desired.module, cwd );
nutils.findOrgs( desired.module, cwd, resolveVersions );
}

var list = [];
orgs.forEach( function( org ){
var versions;

if( desired.version === undefined ){
versions = nutils.findVersions( org, desired.module, cwd );
} else {
versions = nutils.findMatchingVersions( org, desired.module, desired.version, cwd );
}

versions.forEach( function( version ){
list.push( yutils.constructYearningString( { org: ( org === '' ? undefined : org ), module: desired.module, version: version } ) );
} );
} );

callback( null, list );

function resolveVersions( orgs ){
var list = [];

var count = orgs.length;
function done( ){
if( --count <= 0 )
callback( null, list );
}

orgs.forEach( function( org ){

if( desired.version === undefined ){
nutils.findVersions( org, desired.module, cwd, processVersions );
} else {
nutils.findMatchingVersions( org, desired.module, desired.version, cwd, processVersions );
}

function processVersions( versions ){
versions.forEach( function( version ){
list.push( yutils.constructYearningString( { org: ( org === '' ? undefined : org ), module: desired.module, version: version } ) );
} );
done( );
}
} );
}
};

ynpm.commands.shrinkwrap = function( _cwd, _dependencies_seed, _callback ){
ynpm.commands.shrinkwrap = function( _cwd, _dependencies_seed, unsafe, _callback ){

var queue = async.queue( function( task, callback ){
nutils.findMatchingVersions( task.desired.org, task.desired.module, task.desired.version, task.cwd, function( versions ){
callback( null, versions );
} );
}, 8 );

function shrinkwrap_fn( cwd, dependencies_seed, callback ){
function shrinkwrap_fn( cwd, dependencies_seed, dependency_chain, callback ){
var package_location = path.join( cwd, 'package.json' );
var shrinkwrap_location = path.join( cwd, 'ynpm-shrinkwrap.json' );
var pkg, local_shrinkwrap;
Expand Down Expand Up @@ -130,13 +145,34 @@ module.exports = function( config, callback ){
desired.org = ( desired.org === undefined ? '' : desired.org );
desired.version = ( object_semver ? merged_dependencies[ dependency ].version : merged_dependencies[ dependency ] );

desired.version = nutils.findMatchingVersions( desired.org, desired.module, desired.version, cwd )[ 0 ];

var dependency_root = yutils.findModuleLocation( package_location, desired );

shrinkwrap_fn( dependency_root, merged_dependencies[ dependency ].dependencies, function( error, dep_shrinkwrap ){
shrinkwrap.dependencies[ dependency ] = dep_shrinkwrap;
return done();
//nutils.findMatchingVersions( desired.org, desired.module, desired.version, cwd, function( versions ){
queue.push( { desired: desired, cwd: cwd }, function( err, versions ){
if( versions.length === 0 ){
var message = 'Failed to find an acceptable version for ' + yutils.constructYearningString( desired ) + '.';

if( unsafe ){
console.error( message );
return done( );
} else {
throw new Error( message );
}
}

desired.version = versions[ 0 ];
var name = yutils.constructYearningString( desired );
//console.log( 'Adding: ' + yutils.constructYearningString( desired ) );

if( dependency_chain.indexOf( name ) !== -1 ){
shrinkwrap.dependencies[ dependency ] = desired.version;
return done( );
}

yutils.findModuleLocationAsync( package_location, desired, function( dependency_root ){
shrinkwrap_fn( dependency_root, merged_dependencies[ dependency ].dependencies, dependency_chain.concat( name ), function( error, dep_shrinkwrap ){
shrinkwrap.dependencies[ dependency ] = dep_shrinkwrap;
return done();
} );
} );
} );
} );
}
Expand Down Expand Up @@ -164,7 +200,7 @@ module.exports = function( config, callback ){
} );
}

shrinkwrap_fn( _cwd, _dependencies_seed, function( err, shrinkwrap ){
shrinkwrap_fn( _cwd, _dependencies_seed, [], function( err, shrinkwrap ){
if( typeof shrinkwrap === 'object' )
shrinkwrap.name = JSON5.parse( fs.readFileSync( path.join( _cwd, 'package.json' ) ) ).name;

Expand Down
Loading

0 comments on commit 918eabb

Please sign in to comment.