Free memory is taken into account when calculating shard size

lib/file-handling/file-demuxer.js

@@ -7,6 +7,7 @@ var fs = require('fs');
 var EventEmitter = require('events').EventEmitter;
 var merge = require('merge');
 var utils = require('../utils');
+var os = require('os');
 
 /**
  * Takes a single file read stream and outputs several output streams, used for
@@ -141,15 +142,29 @@ FileDemuxer.prototype._needsNewOutputStream = function() {
  */
 FileDemuxer.getOptimalShardSize = function(fileSize, acc) {
   var accumulator = typeof acc === 'undefined' ? 0 : acc;
-  var byteMultiple = (8 * (1024 * 1024)) * Math.pow(2, accumulator);
+
+  // Determine hops back by accumulator
+  var hops = (accumulator - FileDemuxer.SHARD_MULTIPLES_BACK) < 0 ?
+                 0 :
+                 accumulator - FileDemuxer.SHARD_MULTIPLES_BACK;
+
+  // Calculate bytemultiple shard size by hops back
+  var shardSize = function(hops) {
+    return (8 * (1024 * 1024)) * Math.pow(2, hops);
+  };
+
+  var byteMultiple = shardSize(accumulator);
   var check = fileSize / byteMultiple;
 
-  if (check > 0 && check < 2) {
-    var distance = (accumulator - FileDemuxer.SHARD_MULTIPLES_BACK) < 0 ?
-                   0 :
-                   accumulator - FileDemuxer.SHARD_MULTIPLES_BACK;
+  // Determine if bytemultiple is highest bytemultiple that is still <= fileSize
+  if (check > 0 && check <= 1) {
+
+    // TODO: Change 3 to (shard concurrency * file concurrency)
+    while (hops > 0 && (os.freemem() / shardSize(hops) <= 3)){
+      hops = hops - 1 <= 0 ? 0 : hops - 1;
+    }
 
-    return (8 * (1024 * 1024)) * Math.pow(2, distance);
+    return shardSize(hops);
   }
 
   return this.getOptimalShardSize(fileSize, ++accumulator);

test/file-handling/file-demuxer.unit.js

@@ -8,6 +8,7 @@ var os = require('os');
 var fs = require('fs');
 var path = require('path');
 var utils = require('../../lib/utils');
+var proxyquire = require('proxyquire');
 var filePathEven = path.join(os.tmpdir(), 'storjfiledmxtest-even.data');
 var filePathOdd = path.join(os.tmpdir(), 'storjfiledmxtest-odd.data');
 var filePathEmpty = path.join(os.tmpdir(), 'storjfiledmxtest-empty.data');
@@ -114,64 +115,86 @@ describe('FileDemuxer', function() {
 
 describe('FileDemuxer#getOptimalShardSize', function() {
 
+  var FileDemuxerStub = proxyquire('../../lib/file-handling/file-demuxer', {
+    'os': {
+      freemem: function() {
+        return 1024 * (1024 * 1024); //1GB of memory
+      }
+    }
+  });
+
   it('should return 8 for 8', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(8 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(3 * (1024 * 1024))
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 16', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(16 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(16 * (1024 * 1024))
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 32', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(32 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(32 * (1024 * 1024))
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 64', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(64 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(64 * (1024 * 1024))
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 128', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(128 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(128 * (1024 * 1024))
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 8 for 256', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(256 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(256 * (1024 * 1024))
     ).to.equal(8 * (1024 * 1024));
   });
 
   it('should return 16 for 512', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(512 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(512 * (1024 * 1024))
     ).to.equal(16 * (1024 * 1024));
   });
 
   it('should return 32 for 1024', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(1024 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(1024 * (1024 * 1024))
     ).to.equal(32 * (1024 * 1024));
   });
 
   it('should return 64 for 2048', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(2048 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(2048 * (1024 * 1024))
     ).to.equal(64 * (1024 * 1024));
   });
 
   it('should return 128 for 4096', function() {
     expect(
-      FileDemuxer.getOptimalShardSize(4096 * (1024 * 1024))
+      FileDemuxerStub.getOptimalShardSize(4096 * (1024 * 1024))
     ).to.equal(128 * (1024 * 1024));
   });
 
+  it('should return 8 for 4096 if only 16MB of memory', function() {
+    var LowMemDemuxer = proxyquire('../../lib/file-handling/file-demuxer', {
+      'os': {
+        freemem: function() {
+          return 16 * (1024 * 1024); // 16MB of memory
+        }
+      }
+    });
+
+    expect(
+      LowMemDemuxer.getOptimalShardSize(4096 * (1024 * 1024))
+    ).to.equal(8 * (1024 * 1024));
+  });
+
 });