Permalink
Browse files

Serializing the process of the sync, this is since when having many f…

…iles/directories in a single directory, copy operations failed.
  • Loading branch information...
1 parent 1add1b1 commit 75889f905d8308c467d7f50c3f13589a0b72def0 Amit Apple committed Dec 10, 2012
Showing with 127 additions and 61 deletions.
  1. +59 −27 bin/kuduSync.js
  2. +46 −31 lib/FileUtils.ts
  3. +20 −1 lib/Utils.ts
  4. +1 −1 package.json
  5. +1 −1 typings/q.d.ts
View
86 bin/kuduSync.js
@@ -57,6 +57,24 @@ var Utils;
return result;
}
Utils.serialize = serialize;
+ function mapSerialized(source, action) {
+ var result = Q.resolve();
+ for(var i = 0; i < source.length; i++) {
+ var func = {
@pranavkm
pranavkm Dec 10, 2012

Could you use the object declaration for this?

func = { source: ... }?

+ };
+ func.source = source[i];
+ func.i = i;
+ func.action = function () {
+ var self = this;
+ return function () {
+ return action(self.source, self.i);
+ }
+ };
+ result = result.then(func.action());
+ }
+ return result;
+ }
+ Utils.mapSerialized = mapSerialized;
})(Utils || (Utils = {}));
exports.Utils = Utils;
@@ -278,25 +296,39 @@ function copyFile(fromFile, toFilePath, whatIf) {
Ensure.argNotNull(fromFile, "fromFile");
Ensure.argNotNull(toFilePath, "toFilePath");
log("Copying file from: " + fromFile.path() + " to: " + toFilePath);
- return Utils.attempt(function () {
- var deffered = Q.defer();
- try {
- if(!whatIf) {
- var readStream = fs.createReadStream(fromFile.path());
- readStream.pipe(fs.createWriteStream(toFilePath));
- readStream.on("end", function () {
- var toFileStat = fs.statSync(toFilePath);
- fs.utimesSync(toFilePath, toFileStat.atime, fromFile.modifiedTime());
- deffered.resolve();
- });
- } else {
- deffered.resolve();
- }
- return deffered.promise;
- } catch (err) {
- return deffered.reject(err);
- }
- });
+ if(!whatIf) {
+ return Utils.attempt(function () {
+ var promise = copyFileInternal(fromFile, toFilePath);
+ promise = promise.then(function () {
+ return Q.nfcall(fs.stat, toFilePath);
+ });
+ promise = promise.then(function (toFileStat) {
+ return Q.nfcall(fs.utimes, toFilePath, toFileStat.atime, fromFile.modifiedTime());
+ }, null);
+ return promise;
+ });
+ }
+ return Q.resolve();
+}
+function copyFileInternal(fromFile, toFilePath) {
@pranavkm
pranavkm Dec 10, 2012

function copyFileInternal(fromFile, toFilePath) : Promise

Also, I see that the method's repeated twice. Do you think you could move into Utils and share it from there?

EDIT: Nevermind, I realized what I was doing. Ignore these :)

+ var deffered = Q.defer();
+ try {
+ var readStream = fs.createReadStream(fromFile.path());
+ var writeStream = fs.createWriteStream(toFilePath);
+ readStream.pipe(writeStream);
+ readStream.on("end", function () {
+ deffered.resolve();
+ });
+ readStream.on("error", function (err) {
+ deffered.reject(err);
+ });
+ writeStream.on("error", function (err) {
+ deffered.reject(err);
+ });
+ } catch (err) {
+ deffered.reject(err);
+ }
+ return deffered.promise;
}
function deleteFile(file, whatIf) {
Ensure.argNotNull(file, "file");
@@ -362,7 +394,7 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
}, function () {
from.initializeFilesAndSubDirectoriesLists();
}, function () {
- return Q.all(Utils.map(to.filesList(), function (toFile) {
+ return Utils.mapSerialized(to.filesList(), function (toFile) {
if(shouldIgnore(toFile.path(), toRootPath, ignoreList)) {
return Q.resolve();
}
@@ -372,9 +404,9 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
}
}
return Q.resolve();
- }));
+ });
}, function () {
- return Q.all(Utils.map(from.filesList(), function (fromFile) {
+ return Utils.mapSerialized(from.filesList(), function (fromFile) {
if(shouldIgnore(fromFile.path(), fromRootPath, ignoreList)) {
return Q.resolve();
}
@@ -384,21 +416,21 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
return copyFile(fromFile, pathUtil.join(to.path(), fromFile.name()), whatIf);
}
return Q.resolve();
- }));
+ });
}, function () {
- return Q.all(Utils.map(to.subDirectoriesList(), function (toSubDirectory) {
+ return Utils.mapSerialized(to.subDirectoriesList(), function (toSubDirectory) {
if(!from.getSubDirectory(toSubDirectory.name())) {
if(manifest.isPathInManifest(toSubDirectory.path(), toRootPath)) {
return deleteDirectoryRecursive(toSubDirectory, whatIf);
}
}
return Q.resolve();
- }));
+ });
}, function () {
- return Q.all(Utils.map(from.subDirectoriesList(), function (fromSubDirectory) {
+ return Utils.mapSerialized(from.subDirectoriesList(), function (fromSubDirectory) {
var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()));
return kuduSyncDirectory(fromSubDirectory, toSubDirectory, fromRootPath, toRootPath, manifest, outManifest, ignoreList, whatIf);
- }));
+ });
});
} catch (err) {
return Q.reject(err);
View
77 lib/FileUtils.ts
@@ -58,29 +58,44 @@ function copyFile(fromFile: FileInfo, toFilePath: string, whatIf: bool) : Promis
log("Copying file from: " + fromFile.path() + " to: " + toFilePath);
- return Utils.attempt(() => {
- var deffered = Q.defer();
- try {
- if (!whatIf) {
- var readStream = fs.createReadStream(fromFile.path());
- readStream.pipe(fs.createWriteStream(toFilePath));
- readStream.on("end", () => {
- // Update the destination modified time to be the same as the source
- var toFileStat = fs.statSync(toFilePath);
- fs.utimesSync(toFilePath, toFileStat.atime, fromFile.modifiedTime());
- deffered.resolve();
- });
- }
- else {
- deffered.resolve();
- }
-
- return deffered.promise;
- }
- catch (err) {
- return deffered.reject(err);
- }
- });
+ if (!whatIf) {
+ return Utils.attempt(() => {
+ var promise = copyFileInternal(fromFile, toFilePath);
+ promise = promise.then(() => {
+ return Q.nfcall(fs.stat, toFilePath);
+ });
+ promise = promise.then(function (toFileStat?: any) {
+ return Q.nfcall(fs.utimes, toFilePath, toFileStat.atime, fromFile.modifiedTime());
+ }, null);
+
+ return promise;
+ });
+ }
+
+ return Q.resolve();
+}
+
+function copyFileInternal(fromFile: FileInfo, toFilePath: string): Promise {
+ var deffered = Q.defer();
+ try {
+ var readStream = fs.createReadStream(fromFile.path());
+ var writeStream = fs.createWriteStream(toFilePath);
+ readStream.pipe(writeStream);
+ readStream.on("end", () => {
@pranavkm
pranavkm Dec 10, 2012

readStream.on("end", deferred.resolve);
readStream.on("error", deferred.reject);
writeStream.on("error", deferred.reject);

+ deffered.resolve();
+ });
+ readStream.on("error", (err) => {
+ deffered.reject(err);
+ });
+ writeStream.on("error", (err) => {
+ deffered.reject(err);
+ });
+ }
+ catch (err) {
+ deffered.reject(err);
+ }
+
+ return deffered.promise;
}
function deleteFile(file: FileInfo, whatIf: bool) : Promise {
@@ -169,7 +184,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
// If the file doesn't exist in the source, only delete if:
// 1. We have no previous directory
// 2. We have a previous directory and the file exists there
- return Q.all(Utils.map(
+ return Utils.mapSerialized(
to.filesList(),
(toFile: FileInfo) => {
if (shouldIgnore(toFile.path(), toRootPath, ignoreList)) {
@@ -184,12 +199,12 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
}
return Q.resolve();
}
- ));
+ );
},
() => {
// Copy files
- return Q.all(Utils.map(
+ return Utils.mapSerialized(
from.filesList(),
(fromFile: FileInfo) => {
if (shouldIgnore(fromFile.path(), fromRootPath, ignoreList)) {
@@ -209,11 +224,11 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
return Q.resolve();
}
- ));
+ );
},
() => {
- return Q.all(Utils.map(
+ return Utils.mapSerialized(
to.subDirectoriesList(),
(toSubDirectory: DirectoryInfo) => {
// If the file doesn't exist in the source, only delete if:
@@ -226,12 +241,12 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
}
return Q.resolve();
}
- ));
+ );
},
() => {
// Copy directories
- return Q.all(Utils.map(
+ return Utils.mapSerialized(
from.subDirectoriesList(),
(fromSubDirectory: DirectoryInfo) => {
var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()));
@@ -245,7 +260,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
ignoreList,
whatIf);
}
- ));
+ );
});
}
catch (err) {
View
21 lib/Utils.ts
@@ -31,13 +31,32 @@ module Utils {
return results;
}
-
+
export function serialize(...source: {(): Promise; }[]) : Promise {
var result = Q.resolve();
for (var i = 0; i < source.length; i++) {
result = result.then(source[i]);
}
return result;
}
+
+ export function mapSerialized(source: any[], action: (element: any, index: Number) => Promise) : Promise {
+ var result = Q.resolve();
+ for (var i = 0; i < source.length; i++) {
+ var func: any = {};
+ func.source = source[i];
+ func.i = i;
+ func.action = function () {
+ var self = this;
+ return function () {
+ return action(self.source, self.i);
+ }
+ }
+
+ result = result.then(func.action());
+ }
+
+ return result;
+ }
}
exports.Utils = Utils;
View
2 package.json
@@ -1,7 +1,7 @@
{
"name": "kudusync",
"author": "Outercurve Foundation",
- "version": "0.1.10",
+ "version": "0.1.11",
"description": "Tool for syncing files for deployment, will only copy changed files and delete files that doesn't exists in the destination but only if they were part of the previous deployment.",
"tags": [
"azure",
View
2 typings/q.d.ts
@@ -21,7 +21,7 @@ interface QStatic {
reject(value?: any): Promise;
delay(action: Function, delay: Number): Promise;
all(...promises: any[]): Promise;
- nfcall(nodeFunction: Function, thisp: any, ...args: any[]);
+ nfcall(nodeFunction: Function, ...args: any[]);
fcall(nodeFunction: Function, thisp?: any, ...args: any[]);
}

1 comment on commit 75889f9

Please sign in to comment.