Permalink
Browse files

Some more perf improvements, removed unrequired calls.

  • Loading branch information...
1 parent e20309a commit 2c67b33c3bbdebde7204273b9eb7551eedad43ef Amit Apple committed Jan 4, 2013
Showing with 90 additions and 65 deletions.
  1. +35 −25 bin/kuduSync.js
  2. +16 −3 lib/DirectoryInfo.ts
  3. +9 −1 lib/FileInfoBase.ts
  4. +30 −36 lib/FileUtils.ts
View
@@ -124,7 +124,13 @@ var FileInfoBase = (function () {
return this._path;
};
FileInfoBase.prototype.exists = function () {
- return fs.existsSync(this.path());
+ if(!this._exists) {
+ this._exists = fs.existsSync(this.path());
+ }
+ return this._exists;
+ };
+ FileInfoBase.prototype.setExists = function (val) {
+ this._exists = val;
};
return FileInfoBase;
})();
@@ -162,15 +168,22 @@ var DirectoryInfo = (function (_super) {
this._subDirectoriesMapping = [];
this._filesList = [];
this._subDirectoriesList = [];
+ this._initialized = false;
}
DirectoryInfo.prototype.ensureCreated = function () {
var _this = this;
if(!this.exists()) {
- return this.parent().ensureCreated().then(function () {
+ var promise = this.parent().ensureCreated();
+ promise = promise.then(function () {
return Utils.attempt(function () {
return Q.nfcall(fs.mkdir, _this.path());
});
});
+ promise = promise.then(function () {
+ _this.setExists(true);
+ _this._initialized = true;
+ });
+ return promise;
}
return Q.resolve();
};
@@ -183,7 +196,7 @@ var DirectoryInfo = (function (_super) {
var filesList = new Array();
var subDirectoriesMapping = new Array();
var subDirectoriesList = new Array();
- if(this.exists()) {
+ if(!this._initialized && this.exists()) {
return Utils.attempt(function () {
try {
var files = fs.readdirSync(_this.path());
@@ -204,6 +217,7 @@ var DirectoryInfo = (function (_super) {
_this._subDirectoriesMapping = subDirectoriesMapping;
_this._filesList = filesList;
_this._subDirectoriesList = subDirectoriesList;
+ _this._initialized = true;
return Q.resolve();
} catch (err) {
return Q.reject(err);
@@ -283,7 +297,6 @@ var Manifest = (function () {
};
return Manifest;
})();
-var defaultParallelActions = 5;
function kuduSync(fromPath, toPath, nextManifestPath, previousManifestPath, ignore, whatIf) {
Ensure.argNotNull(fromPath, "fromPath");
Ensure.argNotNull(toPath, "toPath");
@@ -333,10 +346,7 @@ function copyFile(fromFile, toFilePath, whatIf) {
return Utils.attempt(function () {
var promise = copyFileInternal(fromFile, toFilePath);
promise = promise.then(function () {
- return Q.nfcall(fs.stat, toFilePath);
- });
- promise = promise.then(function (toFileStat) {
- return Q.nfcall(fs.utimes, toFilePath, toFileStat.atime, fromFile.modifiedTime());
+ return Q.nfcall(fs.utimes, toFilePath, new Date(), fromFile.modifiedTime());
}, null);
return promise;
});
@@ -349,9 +359,9 @@ function copyFileInternal(fromFile, toFilePath) {
var readStream = fs.createReadStream(fromFile.path());
var writeStream = fs.createWriteStream(toFilePath);
readStream.pipe(writeStream);
- readStream.on("end", deffered.resolve);
readStream.on("error", deffered.reject);
writeStream.on("error", deffered.reject);
+ writeStream.on("close", deffered.resolve);
} catch (err) {
deffered.reject(err);
}
@@ -375,10 +385,10 @@ function deleteDirectoryRecursive(directory, whatIf) {
return directory.initializeFilesAndSubDirectoriesLists().then(function () {
var files = directory.filesList();
var subDirectories = directory.subDirectoriesList();
- return Utils.mapParallelized(defaultParallelActions, files, function (file) {
+ return Utils.mapSerialized(files, function (file) {
return deleteFile(file, whatIf);
}).then(function () {
- return Utils.mapParallelized(defaultParallelActions, subDirectories, function (subDir) {
+ return Utils.mapSerialized(subDirectories, function (subDir) {
return deleteDirectoryRecursive(subDir, whatIf);
});
}).then(function () {
@@ -421,18 +431,6 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
}, function () {
return from.initializeFilesAndSubDirectoriesLists();
}, function () {
- return Utils.mapSerialized(to.filesList(), function (toFile) {
- if(shouldIgnore(toFile.path(), toRootPath, ignoreList)) {
- return Q.resolve();
- }
- if(!from.getFile(toFile.name())) {
- if(manifest.isPathInManifest(toFile.path(), toRootPath)) {
- return deleteFile(toFile, whatIf);
- }
- }
- return Q.resolve();
- });
- }, function () {
return Utils.mapSerialized(from.filesList(), function (fromFile) {
if(shouldIgnore(fromFile.path(), fromRootPath, ignoreList)) {
return Q.resolve();
@@ -445,6 +443,18 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
return Q.resolve();
});
}, function () {
+ return Utils.mapSerialized(to.filesList(), function (toFile) {
+ if(shouldIgnore(toFile.path(), toRootPath, ignoreList)) {
+ return Q.resolve();
+ }
+ if(!from.getFile(toFile.name())) {
+ if(manifest.isPathInManifest(toFile.path(), toRootPath)) {
+ return deleteFile(toFile, whatIf);
+ }
+ }
+ return Q.resolve();
+ });
+ }, function () {
return Utils.mapSerialized(to.subDirectoriesList(), function (toSubDirectory) {
if(!from.getSubDirectory(toSubDirectory.name())) {
if(manifest.isPathInManifest(toSubDirectory.path(), toRootPath)) {
@@ -454,10 +464,10 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
return Q.resolve();
});
}, function () {
- return Utils.mapParallelized(defaultParallelActions, from.subDirectoriesList(), function (fromSubDirectory) {
+ return Q.all(Utils.map(from.subDirectoriesList(), function (fromSubDirectory) {
var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()));
return kuduSyncDirectory(fromSubDirectory, toSubDirectory, fromRootPath, toRootPath, manifest, outManifest, ignoreList, whatIf);
- });
+ }));
});
} catch (err) {
return Q.reject(err);
View
@@ -5,6 +5,7 @@ class DirectoryInfo extends FileInfoBase {
private _subDirectoriesMapping: DirectoryInfo[];
private _filesList: FileInfo[];
private _subDirectoriesList: DirectoryInfo[];
+ private _initialized: bool;
constructor (path: string) {
super(path);
@@ -13,14 +14,25 @@ class DirectoryInfo extends FileInfoBase {
this._subDirectoriesMapping = [];
this._filesList = [];
this._subDirectoriesList = [];
+ this._initialized = false;
}
ensureCreated() : Promise {
if (!this.exists()) {
- return this.parent().ensureCreated().then(() => {
+ var promise = this.parent().ensureCreated();
+
+ promise = promise.then(() => {
return Utils.attempt(() => Q.nfcall(fs.mkdir, this.path()));
});
+
+ promise = promise.then(() => {
+ this.setExists(true);
+ this._initialized = true;
+ });
+
+ return promise;
}
+
return Q.resolve();
}
@@ -34,10 +46,9 @@ class DirectoryInfo extends FileInfoBase {
var subDirectoriesMapping = new DirectoryInfo[];
var subDirectoriesList = new DirectoryInfo[];
- if (this.exists()) {
+ if (!this._initialized && this.exists()) {
return Utils.attempt(() => {
try {
- // TODO: Consider changing this call to async
var files = fs.readdirSync(this.path());
files.forEach((fileName: string) => {
var path = pathUtil.join(this.path(), fileName);
@@ -63,6 +74,8 @@ class DirectoryInfo extends FileInfoBase {
this._filesList = filesList;
this._subDirectoriesList = subDirectoriesList;
+ this._initialized = true;
+
return Q.resolve();
}
catch (err) {
View
@@ -3,6 +3,7 @@
class FileInfoBase {
private _name: string;
private _path: string;
+ private _exists: bool;
constructor (path: string) {
Ensure.argNotNull(path, "path");
@@ -20,6 +21,13 @@ class FileInfoBase {
}
exists(): bool {
- return fs.existsSync(this.path());
+ if (!this._exists) {
+ this._exists = fs.existsSync(this.path());
+ }
+ return this._exists;
+ }
+
+ setExists(val: bool) {
+ this._exists = val;
}
}
View
@@ -1,8 +1,6 @@
///<reference path='directoryInfo.ts'/>
///<reference path='manifest.ts'/>
-var defaultParallelActions = 5;
-
function kuduSync(fromPath: string, toPath: string, nextManifestPath: string, previousManifestPath: string, ignore: string, whatIf: bool) : Promise {
Ensure.argNotNull(fromPath, "fromPath");
Ensure.argNotNull(toPath, "toPath");
@@ -63,11 +61,8 @@ function copyFile(fromFile: FileInfo, toFilePath: string, whatIf: bool) : Promis
if (!whatIf) {
return Utils.attempt(() => {
var promise = copyFileInternal(fromFile, toFilePath);
- promise = promise.then(() => {
- return Q.nfcall(fs.stat, toFilePath);
- });
- promise = promise.then(function (toFileStat?: any) {
- return Q.nfcall(fs.utimes, toFilePath, toFileStat.atime, fromFile.modifiedTime());
+ promise = promise.then(function () {
+ return Q.nfcall(fs.utimes, toFilePath, new Date(), fromFile.modifiedTime());
}, null);
return promise;
@@ -83,9 +78,9 @@ function copyFileInternal(fromFile: FileInfo, toFilePath: string): Promise {
var readStream = fs.createReadStream(fromFile.path());
var writeStream = fs.createWriteStream(toFilePath);
readStream.pipe(writeStream);
- readStream.on("end", deffered.resolve);
readStream.on("error", deffered.reject);
writeStream.on("error", deffered.reject);
+ writeStream.on("close", deffered.resolve);
}
catch (err) {
deffered.reject(err);
@@ -120,8 +115,8 @@ function deleteDirectoryRecursive(directory: DirectoryInfo, whatIf: bool) {
var subDirectories = directory.subDirectoriesList();
// Delete all files under this directory
- return Utils.mapParallelized(defaultParallelActions, files, (file) => deleteFile(file, whatIf))
- .then(() => Utils.mapParallelized(defaultParallelActions, subDirectories, (subDir) => deleteDirectoryRecursive(subDir, whatIf)))
+ return Utils.mapSerialized(files, (file) => deleteFile(file, whatIf))
+ .then(() => Utils.mapSerialized(subDirectories, (subDir) => deleteDirectoryRecursive(subDir, whatIf)))
.then(() => {
// Delete current directory
if (!whatIf) {
@@ -167,7 +162,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
}
return Q.resolve();
},
-
+
() => {
return to.initializeFilesAndSubDirectoriesLists();
},
@@ -177,28 +172,6 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
},
() => {
- // If the file doesn't exist in the source, only delete if:
- // 1. We have no previous directory
- // 2. We have a previous directory and the file exists there
- return Utils.mapSerialized(
- to.filesList(),
- (toFile: FileInfo) => {
- if (shouldIgnore(toFile.path(), toRootPath, ignoreList)) {
- // Ignore files in ignore list
- return Q.resolve();
- }
-
- if (!from.getFile(toFile.name())) {
- if (manifest.isPathInManifest(toFile.path(), toRootPath)) {
- return deleteFile(toFile, whatIf);
- }
- }
- return Q.resolve();
- }
- );
- },
-
- () => {
// Copy files
return Utils.mapSerialized(
from.filesList(),
@@ -222,6 +195,28 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
}
);
},
+
+ () => {
+ // If the file doesn't exist in the source, only delete if:
+ // 1. We have no previous directory
+ // 2. We have a previous directory and the file exists there
+ return Utils.mapSerialized(
+ to.filesList(),
+ (toFile: FileInfo) => {
+ if (shouldIgnore(toFile.path(), toRootPath, ignoreList)) {
+ // Ignore files in ignore list
+ return Q.resolve();
+ }
+
+ if (!from.getFile(toFile.name())) {
+ if (manifest.isPathInManifest(toFile.path(), toRootPath)) {
+ return deleteFile(toFile, whatIf);
+ }
+ }
+ return Q.resolve();
+ }
+ );
+ },
() => {
return Utils.mapSerialized(
@@ -242,8 +237,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
() => {
// Copy directories
- return Utils.mapParallelized(
- defaultParallelActions,
+ return Q.all(Utils.map(
from.subDirectoriesList(),
(fromSubDirectory: DirectoryInfo) => {
var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()));
@@ -257,7 +251,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
ignoreList,
whatIf);
}
- );
+ ));
});
}
catch (err) {

0 comments on commit 2c67b33

Please sign in to comment.