Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

kudu service specific performance improvements

  • Loading branch information...
commit 3cb29b807ef45dec42a45196f8adfe0752da8a77 1 parent 31df0b1
@amitapl amitapl authored
View
89 bin/kuduSync.js
@@ -141,25 +141,56 @@ var __extends = this.__extends || function (d, b) {
}
var FileInfo = (function (_super) {
__extends(FileInfo, _super);
- function FileInfo(path, fileStat) {
+ function FileInfo(path, size, modifiedTime) {
_super.call(this, path);
- Ensure.argNotNull(fileStat, "fileStat");
- this._fileStat = fileStat;
+ Ensure.argNotNull(size, "size");
+ Ensure.argNotNull(modifiedTime, "modifiedTime");
+ this._size = size;
+ this._modifiedTime = new Date(modifiedTime);
}
FileInfo.prototype.modifiedTime = function () {
- return this._fileStat.mtime;
+ return this._modifiedTime;
};
FileInfo.prototype.size = function () {
- return this._fileStat.size;
+ return this._size;
};
FileInfo.prototype.equals = function (otherFile) {
- if(this._fileStat == null || otherFile._fileStat == null || this.modifiedTime() == null || otherFile.modifiedTime() == null) {
+ if(this.modifiedTime() == null || otherFile.modifiedTime() == null) {
return false;
}
return this.modifiedTime().getTime() === otherFile.modifiedTime().getTime() && this.size() === otherFile.size();
};
return FileInfo;
})(FileInfoBase);
+var listDir = null;
+try {
+ listDir = require("../ext/fsx_win32").listDir;
+ console.log("Using fsx_win32");
+} catch (e) {
+}
+if(listDir == null) {
+ listDir = function (path) {
+ var files = fs.readdirSync(path);
+ return Utils.map(files, function (fileName) {
+ var filePath = pathUtil.join(path, fileName);
+ var stat = fs.statSync(filePath);
+ if(stat.isDirectory()) {
+ var result = {
+ fileName: fileName,
+ isDirectory: true
+ };
+ return result;
+ } else {
+ var result = {
+ fileName: fileName,
+ size: stat.size,
+ modifiedTime: stat.mtime
+ };
+ return result;
+ }
+ });
+ };
+}
var DirectoryInfo = (function (_super) {
__extends(DirectoryInfo, _super);
function DirectoryInfo(path) {
@@ -199,18 +230,20 @@ var DirectoryInfo = (function (_super) {
if(!this._initialized && this.exists()) {
return Utils.attempt(function () {
try {
- var files = fs.readdirSync(_this.path());
- files.forEach(function (fileName) {
- var path = pathUtil.join(_this.path(), fileName);
- var stat = fs.statSync(path);
- if(stat.isDirectory()) {
- var directoryInfo = new DirectoryInfo(path);
- subDirectoriesMapping[fileName.toUpperCase()] = directoryInfo;
- subDirectoriesList.push(directoryInfo);
- } else {
- var fileInfo = new FileInfo(path, stat);
- filesMapping[fileName.toUpperCase()] = fileInfo;
- filesList.push(fileInfo);
+ var files = listDir(_this.path());
+ files.forEach(function (file) {
+ var path = pathUtil.join(_this.path(), file.fileName);
+ if(file.fileName !== "." && file.fileName !== "..") {
+ if(file.isDirectory) {
+ var directoryInfo = new DirectoryInfo(path);
+ directoryInfo.setExists(true);
+ subDirectoriesMapping[file.fileName.toUpperCase()] = directoryInfo;
+ subDirectoriesList.push(directoryInfo);
+ } else {
+ var fileInfo = new FileInfo(path, file.size, file.modifiedTime);
+ filesMapping[file.fileName.toUpperCase()] = fileInfo;
+ filesList.push(fileInfo);
+ }
}
});
_this._filesMapping = filesMapping;
@@ -303,6 +336,9 @@ function kuduSync(fromPath, toPath, nextManifestPath, previousManifestPath, igno
Ensure.argNotNull(nextManifestPath, "nextManifestPath");
var from = new DirectoryInfo(fromPath);
var to = new DirectoryInfo(toPath);
+ if(!from.exists()) {
+ return Q.reject(new Error("From directory doesn't exist"));
+ }
var nextManifest = new Manifest();
var ignoreList = parseIgnoreList(ignore);
log("Kudu sync from: " + from.path() + " to: " + to.path());
@@ -409,9 +445,6 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
Ensure.argNotNull(manifest, "manifest");
Ensure.argNotNull(outManifest, "outManifest");
try {
- if(!from.exists()) {
- return Q.reject(new Error("From directory doesn't exist"));
- }
if(shouldIgnore(from.path(), fromRootPath, ignoreList)) {
return Q.resolve();
}
@@ -431,7 +464,7 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
}, function () {
return from.initializeFilesAndSubDirectoriesLists();
}, function () {
- return Utils.mapSerialized(from.filesList(), function (fromFile) {
+ return Utils.mapParallelized(5, from.filesList(), function (fromFile) {
if(shouldIgnore(fromFile.path(), fromRootPath, ignoreList)) {
return Q.resolve();
}
@@ -464,10 +497,10 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
return Q.resolve();
});
}, function () {
- return Q.all(Utils.map(from.subDirectoriesList(), function (fromSubDirectory) {
+ return Utils.mapSerialized(from.subDirectoriesList(), function (fromSubDirectory) {
var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()));
return kuduSyncDirectory(fromSubDirectory, toSubDirectory, fromRootPath, toRootPath, manifest, outManifest, ignoreList, whatIf);
- }));
+ });
});
} catch (err) {
return Q.reject(err);
@@ -475,7 +508,7 @@ function kuduSyncDirectory(from, to, fromRootPath, toRootPath, manifest, outMani
}
function main() {
var commander = require("commander");
- commander.version("0.0.1").usage("[options]").option("-f, --fromDir <dir path>", "Source directory to sync").option("-t, --toDir <dir path>", "Destination directory to sync").option("-n, --nextManifest <manifest file path>", "Next manifest file path").option("-p, --previousManifest [manifest file path]", "Previous manifest file path").option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;").option("-q, --quiet", "No logging").option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines").option("-w, --whatIf", "Only log without actual copy/remove of files").parse(process.argv);
+ commander.version("0.0.1").usage("[options]").option("-f, --fromDir <dir path>", "Source directory to sync").option("-t, --toDir <dir path>", "Destination directory to sync").option("-n, --nextManifest <manifest file path>", "Next manifest file path").option("-p, --previousManifest [manifest file path]", "Previous manifest file path").option("-i, --ignore [patterns]", "List of files/directories to ignore and not sync, delimited by ;").option("-q, --quiet", "No logging").option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines").option("-w, --whatIf", "Only log without actual copy/remove of files").option("--perf", "Print out the time it took to complete KuduSync operation").parse(process.argv);
var commanderValues = commander;
var fromDir = commanderValues.fromDir;
var toDir = commanderValues.toDir;
@@ -485,6 +518,7 @@ function main() {
var quiet = commanderValues.quiet;
var verbose = commanderValues.verbose;
var whatIf = commanderValues.whatIf;
+ var perf = commanderValues.perf;
if(quiet && verbose) {
console.log("Error: Cannot use --quiet and --verbose arguments together");
process.exit(1);
@@ -513,7 +547,12 @@ function main() {
counter++;
};
}
+ var start = new Date();
kuduSync(fromDir, toDir, nextManifest, previousManifest, ignore, whatIf).then(function () {
+ if(perf) {
+ var stop = new Date();
+ console.log("Operation took " + ((stop.getTime() - start.getTime()) / 1000) + " seconds");
+ }
process.exit(0);
}, function (err) {
if(err) {
View
BIN  ext/fsx_win32.node
Binary file not shown
View
59 lib/DirectoryInfo.ts
@@ -1,5 +1,40 @@
///<reference path='fileInfo.ts'/>
+var listDir: (path: string) => any[] = null;
+
+try {
+ listDir = require("../ext/fsx_win32").listDir;
+ console.log("Using fsx_win32");
+}
+catch (e) {
+}
+
+if (listDir == null) {
+ listDir = function (path: string) {
+ var files = fs.readdirSync(path);
+ return Utils.map(files, (fileName) => {
+ var filePath = pathUtil.join(path, fileName);
+ var stat = fs.statSync(filePath);
+
+ if (stat.isDirectory()) {
+ var result: any = {
+ fileName: fileName,
+ isDirectory: true
+ };
+ return result;
+ }
+ else {
+ var result: any = {
+ fileName: fileName,
+ size: stat.size,
+ modifiedTime: stat.mtime
+ };
+ return result;
+ }
+ });
+ }
+}
+
class DirectoryInfo extends FileInfoBase {
private _filesMapping: FileInfo[];
private _subDirectoriesMapping: DirectoryInfo[];
@@ -40,7 +75,7 @@ class DirectoryInfo extends FileInfoBase {
return new DirectoryInfo(pathUtil.dirname(this.path()));
}
- initializeFilesAndSubDirectoriesLists() : Promise {
+ initializeFilesAndSubDirectoriesLists(): Promise {
var filesMapping = new FileInfo[];
var filesList = new FileInfo[];
var subDirectoriesMapping = new DirectoryInfo[];
@@ -49,25 +84,25 @@ class DirectoryInfo extends FileInfoBase {
if (!this._initialized && this.exists()) {
return Utils.attempt(() => {
try {
- var files = fs.readdirSync(this.path());
- files.forEach((fileName: string) => {
- var path = pathUtil.join(this.path(), fileName);
- var stat = fs.statSync(path);
+ var files = listDir(this.path());
+ files.forEach((file: any) => {
+ var path = pathUtil.join(this.path(), file.fileName);
- if (stat.isDirectory()) {
+ if (file.fileName !== "." && file.fileName !== "..") {
+ if (file.isDirectory) {
// Store both as mapping as an array
var directoryInfo = new DirectoryInfo(path);
- subDirectoriesMapping[fileName.toUpperCase()] = directoryInfo;
+ directoryInfo.setExists(true);
+ subDirectoriesMapping[file.fileName.toUpperCase()] = directoryInfo;
subDirectoriesList.push(directoryInfo);
- }
- else {
+ } else {
// Store both as mapping as an array
- var fileInfo = new FileInfo(path, stat);
- filesMapping[fileName.toUpperCase()] = fileInfo;
+ var fileInfo = new FileInfo(path, file.size, file.modifiedTime);
+ filesMapping[file.fileName.toUpperCase()] = fileInfo;
filesList.push(fileInfo);
}
}
- );
+ });
this._filesMapping = filesMapping;
this._subDirectoriesMapping = subDirectoriesMapping;
View
18 lib/FileInfo.ts
@@ -1,26 +1,28 @@
///<reference path='fileInfoBase.ts'/>
class FileInfo extends FileInfoBase {
- private _fileStat: FileStat;
+ private _size;
+ private _modifiedTime: Date;
- constructor (path: string, fileStat: FileStat) {
+ constructor (path: string, size: any, modifiedTime: any) {
super(path);
+ Ensure.argNotNull(size, "size");
+ Ensure.argNotNull(modifiedTime, "modifiedTime");
- Ensure.argNotNull(fileStat, "fileStat");
-
- this._fileStat = fileStat;
+ this._size = size;
+ this._modifiedTime = new Date(modifiedTime);
}
modifiedTime() {
- return this._fileStat.mtime;
+ return this._modifiedTime;
}
size() {
- return this._fileStat.size;
+ return this._size;
}
equals(otherFile: FileInfo): bool {
- if (this._fileStat == null || otherFile._fileStat == null || this.modifiedTime() == null || otherFile.modifiedTime() == null) {
+ if (this.modifiedTime() == null || otherFile.modifiedTime() == null) {
return false;
}
View
25 lib/FileUtils.ts
@@ -9,6 +9,10 @@ function kuduSync(fromPath: string, toPath: string, nextManifestPath: string, pr
var from = new DirectoryInfo(fromPath);
var to = new DirectoryInfo(toPath);
+ if (!from.exists()) {
+ return Q.reject(new Error("From directory doesn't exist"));
+ }
+
var nextManifest = new Manifest();
var ignoreList = parseIgnoreList(ignore);
@@ -135,11 +139,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
Ensure.argNotNull(manifest, "manifest");
Ensure.argNotNull(outManifest, "outManifest");
- try {
- if (!from.exists()) {
- return Q.reject(new Error("From directory doesn't exist"));
- }
-
+ try {
if (shouldIgnore(from.path(), fromRootPath, ignoreList)) {
// Ignore directories in ignore list
return Q.resolve();
@@ -164,16 +164,17 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
},
() => {
- return to.initializeFilesAndSubDirectoriesLists();
+ return to.initializeFilesAndSubDirectoriesLists();
},
() => {
- return from.initializeFilesAndSubDirectoriesLists();
+ return from.initializeFilesAndSubDirectoriesLists();
},
() => {
// Copy files
- return Utils.mapSerialized(
+ return Utils.mapParallelized(
+ 5,
from.filesList(),
(fromFile: FileInfo) => {
if (shouldIgnore(fromFile.path(), fromRootPath, ignoreList)) {
@@ -202,7 +203,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
// 2. We have a previous directory and the file exists there
return Utils.mapSerialized(
to.filesList(),
- (toFile: FileInfo) => {
+ (toFile: FileInfo) => {
if (shouldIgnore(toFile.path(), toRootPath, ignoreList)) {
// Ignore files in ignore list
return Q.resolve();
@@ -237,7 +238,7 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
() => {
// Copy directories
- return Q.all(Utils.map(
+ return Utils.mapSerialized(
from.subDirectoriesList(),
(fromSubDirectory: DirectoryInfo) => {
var toSubDirectory = new DirectoryInfo(pathUtil.join(to.path(), fromSubDirectory.name()));
@@ -251,10 +252,10 @@ function kuduSyncDirectory(from: DirectoryInfo, to: DirectoryInfo, fromRootPath:
ignoreList,
whatIf);
}
- ));
+ );
});
}
catch (err) {
return Q.reject(err);
- }
+ }
}
View
7 lib/Main.ts
@@ -15,6 +15,7 @@ function main() {
.option("-q, --quiet", "No logging")
.option("-v, --verbose [maxLines]", "Verbose logging with maximum number of output lines")
.option("-w, --whatIf", "Only log without actual copy/remove of files")
+ .option("--perf", "Print out the time it took to complete KuduSync operation")
.parse(process.argv);
var commanderValues: any = commander;
@@ -26,6 +27,7 @@ function main() {
var quiet = commanderValues.quiet;
var verbose = commanderValues.verbose;
var whatIf = commanderValues.whatIf;
+ var perf = commanderValues.perf;
if (quiet && verbose) {
console.log("Error: Cannot use --quiet and --verbose arguments together");
@@ -64,6 +66,7 @@ function main() {
};
}
+ var start = new Date();
kuduSync(
fromDir,
toDir,
@@ -72,6 +75,10 @@ function main() {
ignore,
whatIf).then(
() => {
+ if (perf) {
+ var stop = new Date();
+ console.log("Operation took " + ((stop.getTime() - start.getTime()) / 1000) + " seconds");
+ }
process.exit(0);
},
function (err?) {
Please sign in to comment.
Something went wrong with that request. Please try again.