Fix Poloniex not fetching all available trades (#1565) (#1661)

commands/backfill.js

@@ -37,6 +37,7 @@ module.exports = function (program, conf) {
       var target_time, start_time
       var mode = exchange.historyScan
       var last_batch_id, last_batch_opts
+      var offset = exchange.offset
       var markers, trades
       if (!mode) {
         console.error('cannot backfill ' + selector.normalized + ': exchange does not offer historical data')
@@ -74,6 +75,9 @@ module.exports = function (program, conf) {
           if (marker.to) opts.from = marker.to + 1
           else opts.from = exchange.getCursor(start_time)
         }
+        if (offset) {
+          opts.offset = offset
+        }
         last_batch_opts = opts
         exchange.getTrades(opts, function (err, results) {
           trades = results

extensions/exchanges/poloniex/exchange.js

@@ -40,6 +40,7 @@ module.exports = function container (conf) {
     historyScan: 'backward',
     makerFee: 0.15,
     takerFee: 0.25,
+    offset: 43200,
 
     getProducts: function () {
       return require('./products.json')
@@ -59,11 +60,11 @@ module.exports = function container (conf) {
       }
       if (args.start && !args.end) {
         // add 12 hours
-        args.end = args.start + 43200
+        args.end = args.start + opts.offset
       }
       else if (args.end && !args.start) {
         // subtract 12 hours
-        args.start = args.end - 43200
+        args.start = args.end - opts.offset
       }
 
       client._public('returnTradeHistory', args, function (err, body) {
@@ -76,6 +77,12 @@ module.exports = function container (conf) {
           console.error(body)
           return retry('getTrades', func_args)
         }
+
+        if (body.length >= 50000) {
+          func_args[0].offset = opts.offset / 2;
+          return retry('getTrades', func_args)
+        }
+
         var trades = body.map(function (trade) {
           return {
             trade_id: trade.tradeID,