Shard concurrency taken into account when determining shard size

lib/bridge-client/index.js

@@ -395,7 +395,12 @@ BridgeClient.prototype.storeFileInBucket = function(id, token, file, cb) {
     return cb(new Error(fileSize +' bytes is not a supported file size.'));
   }
 
-  var shardSize = FileDemuxer.getOptimalShardSize(fileSize);
+  var shardSize = FileDemuxer.getOptimalShardSize(
+    {
+      fileSize: fileSize,
+      shardConcurrency: this._transferConcurrency
+    }
+  );
   var uploadState = new UploadState({
     id: id,
     file: file,

lib/file-handling/file-demuxer.js

@@ -136,11 +136,13 @@ FileDemuxer.prototype._needsNewOutputStream = function() {
 
 /**
  * Determine the optimal shard size given an arbitrary file size in bytes
- * @param {Number} fileSize - The number of bytes in the given file
+ * @param {Object} fileInfo
+ * @param {Number} fileInfo.fileSize - The number of bytes in the given file
+ * @param {Number} fileInfo.shardConcurrency - Num of shards uploaded at once
  * @param {Number} [acc=1] - Accumulator (number of recursions)
  * @returns {Number} shardSize
  */
-FileDemuxer.getOptimalShardSize = function(fileSize, acc) {
+FileDemuxer.getOptimalShardSize = function(fileInfo, acc) {
   var accumulator = typeof acc === 'undefined' ? 0 : acc;
 
   // Determine hops back by accumulator
@@ -154,20 +156,23 @@ FileDemuxer.getOptimalShardSize = function(fileSize, acc) {
   };
 
   var byteMultiple = shardSize(accumulator);
-  var check = fileSize / byteMultiple;
+  var check = fileInfo.fileSize / byteMultiple;
 
   // Determine if bytemultiple is highest bytemultiple that is still <= fileSize
   if (check > 0 && check <= 1) {
 
-    // TODO: Change 3 to (shard concurrency * file concurrency)
-    while (hops > 0 && (os.freemem() / shardSize(hops) <= 3)){
+    // Certify the number of concurrency * shardSize doesn't exceed freemem
+    while (
+      hops > 0 &&
+      (os.freemem() / shardSize(hops) <= fileInfo.shardConcurrency)
+    ) {
       hops = hops - 1 <= 0 ? 0 : hops - 1;
     }
 
     return shardSize(hops);
   }
 
-  return this.getOptimalShardSize(fileSize, ++accumulator);
+  return this.getOptimalShardSize(fileInfo, ++accumulator);
 };
 
 module.exports = FileDemuxer;

test/file-handling/file-demuxer.unit.js

@@ -125,61 +125,111 @@ describe('FileDemuxer#getOptimalShardSize', function() {
 
   it('should return 8 for 8', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(3 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 3 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 16', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(16 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 16 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 32', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(32 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 32 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 64', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(64 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 64 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 128', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(128 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 128 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 256', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(256 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 256 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 16 for 512', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(512 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 512 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(16 * (1024 * 1024));
   });
 
   it('should return 32 for 1024', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(1024 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 1024 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(32 * (1024 * 1024));
   });
 
   it('should return 64 for 2048', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(2048 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 2048 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(64 * (1024 * 1024));
   });
 
   it('should return 128 for 4096', function() {
     expect(
-      FileDemuxerStub.getOptimalShardSize(4096 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(
+        {
+          fileSize: 4096 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(128 * (1024 * 1024));
   });
 
@@ -193,7 +243,12 @@ describe('FileDemuxer#getOptimalShardSize', function() {
     });
 
     expect(
-      LowMemDemuxer.getOptimalShardSize(4096 * (1024 * 1024))
+      LowMemDemuxer.getOptimalShardSize(
+        {
+          fileSize: 4096 * (1024 * 1024),
+          shardConcurrency: 3
+        }
+      )
     ).to.equal(8 * (1024 * 1024));
   });