Skip to content

Commit

Permalink
Merge pull request #359 from wesleytodd/bulk-copy
Browse files Browse the repository at this point in the history 
Bulk copy feature #350
  • Loading branch information
@sindresorhus
sindresorhus committed Oct 19, 2013
2 parents 4b330bb + a3a2400 commit 24d93b5
Show file tree
Hide file tree
Showing 3 changed files with 195 additions and 56 deletions.
149 changes: 108 additions & 41 deletions lib/actions/actions.js
View file
Expand Up @@ -72,17 +72,8 @@ actions.cacheRoot = function cacheRoot() {
return path.join(home, '.cache/yeoman');
};

/**
* Make some of the file API aware of our source/destination root paths.
* `copy`, `template` (only when could be applied/required by legacy code),
* `write` and alike consider.
*
* @param {String} source
* @param {String} destination
* @param {Function} process
*/

actions.copy = function copy(source, destination, process) {
// Copy helper for two versions of copy action
function prepCopy(source, destination, process) {
var body;
destination = destination || source;

Expand All @@ -107,14 +98,36 @@ actions.copy = function copy(source, destination, process) {
});
}

return {
body: body,
encoding: encoding,
destination: destination,
source: source
};
};

/**
* Make some of the file API aware of our source/destination root paths.
* `copy`, `template` (only when could be applied/required by legacy code),
* `write` and alike consider.
*
* @param {String} source
* @param {String} destination
* @param {Function} process
*/

actions.copy = function copy(source, destination, process) {

var file = prepCopy.call(this, source, destination, process);

try {
body = this.engine(body, this);
file.body = this.engine(file.body, this);
} catch (err) {
// this happens in some cases when trying to copy a JS file like lodash/underscore
// (conflicting the templating engine)
}

this.checkForCollision(destination, body, function (err, config) {
this.checkForCollision(file.destination, file.body, function (err, config) {
var stats;

if (err) {
Expand All @@ -128,16 +141,16 @@ actions.copy = function copy(source, destination, process) {
return config.callback();
}

mkdirp.sync(path.dirname(destination));
fs.writeFileSync(destination, body);
mkdirp.sync(path.dirname(file.destination));
fs.writeFileSync(file.destination, file.body);

// synchronize stats and modification times from the original file.
stats = fs.statSync(source);
stats = fs.statSync(file.source);
try {
fs.chmodSync(destination, stats.mode);
fs.utimesSync(destination, stats.atime, stats.mtime);
fs.chmodSync(file.destination, stats.mode);
fs.utimesSync(file.destination, stats.atime, stats.mtime);
} catch (err) {
this.log.error('Error setting permissions of "' + chalk.bold(destination) + '" file: ' + err);
this.log.error('Error setting permissions of "' + chalk.bold(file.destination) + '" file: ' + err);
}

config.callback();
Expand All @@ -146,6 +159,39 @@ actions.copy = function copy(source, destination, process) {
return this;
};

/**
* Bulk copy
* https://github.com/yeoman/generator/pull/359
* https://github.com/yeoman/generator/issues/350
*
* An optimized copy method for larger file trees. Does not do
* full conflicter checks, only check ir root directory is not empty.
*
* @param {String} source
* @param {String} destination
* @param {Function} process
*/

actions.bulkCopy = function bulkCopy(source, destination, process) {

var file = prepCopy.call(this, source, destination, process);

mkdirp.sync(path.dirname(file.destination));
fs.writeFileSync(file.destination, file.body);

// synchronize stats and modification times from the original file.
stats = fs.statSync(file.source);
try {
fs.chmodSync(file.destination, stats.mode);
fs.utimesSync(file.destination, stats.atime, stats.mtime);
} catch (err) {
this.log.error('Error setting permissions of "' + chalk.bold(file.destination) + '" file: ' + err);
}

log.create(file.destination);
return this;
};

/**
* A simple method to read the content of the a file borrowed from Grunt:
* https://github.com/gruntjs/grunt/blob/master/lib/grunt/file.js
Expand Down Expand Up @@ -292,18 +338,10 @@ actions.engine = function engine(body, data) {
body;
};

/**
* Copies recursively the files from source directory to root directory.
*
* @param {String} source
* @param {String} destination
* @param {Function} process
*/

actions.directory = function directory(source, destination, process) {
// Shared directory method
function _directory(source, destination, process, bulk) {
var root = path.join(this.sourceRoot(), source);
var files = this.expandFiles('**', { dot: true, cwd: root });
var self = this;

destination = destination || source;

Expand All @@ -312,23 +350,52 @@ actions.directory = function directory(source, destination, process) {
destination = source;
}

var cp = this.copy;
if (bulk) {
cp = this.bulkCopy;
}

// get the path relative to the template root, and copy to the relative destination
var resolveFiles = function (filepath) {
return function (next) {
if (!filepath) {
self.emit('directory:end');
return next();
}
for (var i in files) {
var dest = path.join(destination, files[i]);
cp.call(this, path.join(root, files[i]), dest, process);
}

var dest = path.join(destination, filepath);
self.copy(path.join(root, filepath), dest, process);
return this;
};

return next();
};
};
/**
* Copies recursively the files from source directory to root directory.
*
* @param {String} source
* @param {String} destination
* @param {Function} process
*/

actions.directory = function directory(source, destination, process) {
return _directory.call(this, source, destination, process);
};

/**
* Copies recursively the files from source directory to root directory.
*
* @param {String} source
* @param {String} destination
* @param {Function} process
*/

async.parallel(files.map(resolveFiles));
actions.bulkDirectory = function directory(source, destination, process) {
var self = this;
this.checkForCollision(destination, null, function (err, config) {
// create or force means file write, identical or skip prevent the
// actual write.
if (!(/force|create/.test(config.status))) {
return config.callback();
}

_directory.call(this, source, destination, process, true);
config.callback();
});
return this;
};

Expand Down
32 changes: 17 additions & 15 deletions lib/util/conflicter.js
View file
Expand Up @@ -143,22 +143,24 @@ conflicter.collision = function collision(filepath, content, cb) {
return cb('create');
}

var encoding = null;
if (!isBinaryFile(path.resolve(filepath))) {
encoding = 'utf8';
}
if (!fs.statSync(path.resolve(filepath)).isDirectory()) {
var encoding = null;
if (!isBinaryFile(path.resolve(filepath))) {
encoding = 'utf8';
}

var actual = fs.readFileSync(path.resolve(filepath), encoding);

// In case of binary content, `actual` and `content` are `Buffer` objects,
// we just can't compare those 2 objects with standard `===`,
// so we convert each binary content to an hexadecimal string first, and then compare them with standard `===`
//
// For not binary content, we can directly compare the 2 strings this way
if ((!encoding && (actual.toString('hex') === content.toString('hex'))) ||
(actual === content)) {
log.identical(filepath);
return cb('identical');
var actual = fs.readFileSync(path.resolve(filepath), encoding);

// In case of binary content, `actual` and `content` are `Buffer` objects,
// we just can't compare those 2 objects with standard `===`,
// so we convert each binary content to an hexadecimal string first, and then compare them with standard `===`
//
// For not binary content, we can directly compare the 2 strings this way
if ((!encoding && (actual.toString('hex') === content.toString('hex'))) ||
(actual === content)) {
log.identical(filepath);
return cb('identical');
}
}

if (self.force) {
Expand Down
70 changes: 70 additions & 0 deletions test/actions.js
View file
Expand Up @@ -131,6 +131,37 @@ describe('yeoman.generators.Base', function () {
});
});

describe('generator.bulkCopy(source, destination, process)', function () {

before(function (done) {
this.dummy.bulkCopy(path.join(__dirname, 'fixtures/foo.js'), 'write/to/foo.js');
this.dummy.bulkCopy(path.join(__dirname, 'fixtures/foo-template.js'), 'write/to/noProcess.js');
done();
});

it('should copy a file', function (done) {
fs.readFile('write/to/foo.js', function (err, data) {
if (err) throw err;
assert.equal(data+'', 'var foo = \'foo\';\n');
done();
});
});

it('should not run conflicter or template engine', function (done) {
var self = this;
fs.readFile('write/to/noProcess.js', function (err, data) {
if (err) throw err;
assert.equal(data+'', 'var <%= foo %> = \'<%= foo %>\';\n');
self.dummy.bulkCopy(path.join(__dirname, 'fixtures/foo.js'), 'write/to/noProcess.js');
fs.readFile('write/to/noProcess.js', function (err, data) {
if (err) throw err;
assert.equal(data+'', 'var foo = \'foo\';\n');
done();
});
});
});
});

describe('generator.read(filepath, encoding)', function () {
it('should read files relative to the "sourceRoot" value', function () {
var body = this.dummy.read('foo.js');
Expand Down Expand Up @@ -251,6 +282,45 @@ describe('yeoman.generators.Base', function () {
var body = fs.readFileSync('directory-processed/foo-process.js', 'utf8');
helpers.assertTextEqual(body, 'var bar = \'foo\';\n');
});

});

describe('generator.bulkDirectory(source, destination, process)', function () {
before(function (done) {
this.dummy.sourceRoot(this.fixtures);
this.dummy.destinationRoot('.');
// Create temp bulk operation files
// These cannot just be in the repo or the other directory tests fail
require('mkdirp').sync(this.fixtures + '/bulk-operation');
for (var i = 0; i < 1000; i++) {
fs.writeFileSync(this.fixtures + '/bulk-operation/' + i + '.js', i);
}
// Copy files without processing
this.dummy.bulkDirectory('bulk-operation', 'bulk-operation');
this.dummy.conflicter.resolve(done);
});

after(function (done) {
// Now remove them
for (var i = 0; i < 1000; i++) {
fs.unlinkSync(this.fixtures + '/bulk-operation/' + i + '.js');
}
fs.rmdirSync(this.fixtures + '/bulk-operation');
done();
});

it('should bulk copy one thousand files', function (done) {
fs.readFile('bulk-operation/999.js', function (err, data) {
if (err) throw err;
assert.equal(data, '999');
done();
});
});

it('should check for conflict if directory already exists', function (done) {
this.dummy.bulkDirectory('bulk-operation', 'bulk-operation');
this.dummy.conflicter.resolve(done);
});
});

describe('actions/install', function () {
Expand Down

0 comments on commit 24d93b5

Please sign in to comment.