From 654dc3677d65d43447c679ee4f35622d17253c8e Mon Sep 17 00:00:00 2001 From: aleitner Date: Fri, 9 Sep 2016 12:20:54 -0400 Subject: [PATCH] Shard concurrency taken into account when determining shard size --- lib/bridge-client/index.js | 7 ++- lib/file-handling/file-demuxer.js | 17 ++++-- test/file-handling/file-demuxer.unit.js | 77 +++++++++++++++++++++---- 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/lib/bridge-client/index.js b/lib/bridge-client/index.js index 0a1b2145..0cd4df64 100644 --- a/lib/bridge-client/index.js +++ b/lib/bridge-client/index.js @@ -395,7 +395,12 @@ BridgeClient.prototype.storeFileInBucket = function(id, token, file, cb) { return cb(new Error(fileSize +' bytes is not a supported file size.')); } - var shardSize = FileDemuxer.getOptimalShardSize(fileSize); + var shardSize = FileDemuxer.getOptimalShardSize( + { + fileSize: fileSize, + shardConcurrency: this._transferConcurrency + } + ); var uploadState = new UploadState({ id: id, file: file, diff --git a/lib/file-handling/file-demuxer.js b/lib/file-handling/file-demuxer.js index e4909015..9353dd6a 100644 --- a/lib/file-handling/file-demuxer.js +++ b/lib/file-handling/file-demuxer.js @@ -136,11 +136,13 @@ FileDemuxer.prototype._needsNewOutputStream = function() { /** * Determine the optimal shard size given an arbitrary file size in bytes - * @param {Number} fileSize - The number of bytes in the given file + * @param {Object} fileInfo + * @param {Number} fileInfo.fileSize - The number of bytes in the given file + * @param {Number} fileInfo.shardConcurrency - Num of shards uploaded at once * @param {Number} [acc=1] - Accumulator (number of recursions) * @returns {Number} shardSize */ -FileDemuxer.getOptimalShardSize = function(fileSize, acc) { +FileDemuxer.getOptimalShardSize = function(fileInfo, acc) { var accumulator = typeof acc === 'undefined' ? 0 : acc; // Determine hops back by accumulator @@ -154,20 +156,23 @@ FileDemuxer.getOptimalShardSize = function(fileSize, acc) { }; var byteMultiple = shardSize(accumulator); - var check = fileSize / byteMultiple; + var check = fileInfo.fileSize / byteMultiple; // Determine if bytemultiple is highest bytemultiple that is still <= fileSize if (check > 0 && check <= 1) { - // TODO: Change 3 to (shard concurrency * file concurrency) - while (hops > 0 && (os.freemem() / shardSize(hops) <= 3)){ + // Certify the number of concurrency * shardSize doesn't exceed freemem + while ( + hops > 0 && + (os.freemem() / shardSize(hops) <= fileInfo.shardConcurrency) + ) { hops = hops - 1 <= 0 ? 0 : hops - 1; } return shardSize(hops); } - return this.getOptimalShardSize(fileSize, ++accumulator); + return this.getOptimalShardSize(fileInfo, ++accumulator); }; module.exports = FileDemuxer; diff --git a/test/file-handling/file-demuxer.unit.js b/test/file-handling/file-demuxer.unit.js index 9738420b..8bcf6dd5 100644 --- a/test/file-handling/file-demuxer.unit.js +++ b/test/file-handling/file-demuxer.unit.js @@ -125,61 +125,111 @@ describe('FileDemuxer#getOptimalShardSize', function() { it('should return 8 for 8', function() { expect( - FileDemuxerStub.getOptimalShardSize(3 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 3 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(8 * (1024 * 1024)); }); it('should return 8 for 16', function() { expect( - FileDemuxerStub.getOptimalShardSize(16 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 16 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(8 * (1024 * 1024)); }); it('should return 8 for 32', function() { expect( - FileDemuxerStub.getOptimalShardSize(32 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 32 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(8 * (1024 * 1024)); }); it('should return 8 for 64', function() { expect( - FileDemuxerStub.getOptimalShardSize(64 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 64 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(8 * (1024 * 1024)); }); it('should return 8 for 128', function() { expect( - FileDemuxerStub.getOptimalShardSize(128 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 128 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(8 * (1024 * 1024)); }); it('should return 8 for 256', function() { expect( - FileDemuxerStub.getOptimalShardSize(256 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 256 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(8 * (1024 * 1024)); }); it('should return 16 for 512', function() { expect( - FileDemuxerStub.getOptimalShardSize(512 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 512 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(16 * (1024 * 1024)); }); it('should return 32 for 1024', function() { expect( - FileDemuxerStub.getOptimalShardSize(1024 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 1024 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(32 * (1024 * 1024)); }); it('should return 64 for 2048', function() { expect( - FileDemuxerStub.getOptimalShardSize(2048 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 2048 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(64 * (1024 * 1024)); }); it('should return 128 for 4096', function() { expect( - FileDemuxerStub.getOptimalShardSize(4096 * (1024 * 1024)) + FileDemuxerStub.getOptimalShardSize( + { + fileSize: 4096 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(128 * (1024 * 1024)); }); @@ -193,7 +243,12 @@ describe('FileDemuxer#getOptimalShardSize', function() { }); expect( - LowMemDemuxer.getOptimalShardSize(4096 * (1024 * 1024)) + LowMemDemuxer.getOptimalShardSize( + { + fileSize: 4096 * (1024 * 1024), + shardConcurrency: 3 + } + ) ).to.equal(8 * (1024 * 1024)); });