Resampling after Backfill?

[ghost]

Thank you again for this great software and for making it open source! Thats greatly appreciated!!
I have noticed that the system does not resample after a backfill, causing in my case the system not having enough candlesticks in the database for my indicator.
Per what I saw the resampling only happens during official trading, and only for the last 750 results from the exchange (Bitmex in my case).
Backfill currently is only possible for Bitmex with the binsizes from Bitmex, no resampling values possible.

Maybe it would be a good enhancement to have the resampling run after the backfill as well, or am I making a mistake here?

[ghost]

I have implemented this in the modules/backfill.js. If you like it you could use it for the main master.

Whole file:

'use strict';

var moment = require('moment');
let CandlestickEvent = require('../event/candlestick_event')
var _ = require('lodash')
let resample = require('./../utils/resample')

module.exports = class Backfill {
    constructor(exchangesIterator, candleStickListener) {
        this.exchangesIterator = exchangesIterator
        this.candleStickListener = candleStickListener
    }

    async backfill(exchangeName, symbol, period, date) {
        let exchange = this.exchangesIterator.find(e => e.getName() === exchangeName)
        if (!exchange) {
            throw 'Exchange not found: ' + exchangeName
        }

        let start = moment().subtract(date, 'days');
        let results = undefined
        let totalResults = []

        do {
            console.log('Since: ' + start)
            results = await exchange.backfill(symbol, period, start)
            totalResults = totalResults.concat(results)

            let event = new CandlestickEvent(exchangeName, symbol, period, results)

            await this.candleStickListener.onCandleStick(event)

            console.log('Got: ' + results.length)

            start = _.max(results.map(r => new Date(r.time * 1000)))
        } while (results.length > 10);

        // @TODO This should be exported into a customizing/global config? 
        // Same as in the exchanges, e.g. bitmex.js
        let resamples = {};

        if (!resamples[symbol.symbol]) {
            resamples[symbol.symbol] = {}
        }

        if (!resamples[symbol.symbol]['5m']) {
            resamples[symbol.symbol]['5m'] = []
        }

        resamples[symbol.symbol]['5m'].push('15m')

        // lets wait for settle down of database insert: per design we dont know when is was inserted to database
        if (resamples[symbol['symbol']] && resamples[symbol['symbol']][period] && resamples[symbol['symbol']][period].length > 0) {
            console.log('Start Resampling')
            setTimeout(async () => {
                for (let periodTo of resamples[symbol['symbol']][period]) {
                    let resampledCandles = resample.resampleMinutes(
                        totalResults.reverse().slice(),
                        resample.convertPeriodToMinute(periodTo)
                    )
                    let event = new CandlestickEvent(exchangeName, symbol, periodTo, resampledCandles)

                    await this.candleStickListener.onCandleStick(event)

                    console.log('Resampling: ' + period + ' -> ' + periodTo)
                }
                console.log('Resampling Finished')

            }, 1000);
        }
    }
};

[ghost]

The referenced commit does not include a resampling of the backfilled data, does it?

[Haehnchen]

no, but resulted in that not all resampled data was saved

[ghost]

Ah, I see okay, thanks!

I have update my coding for this. Feel free to use it, if you like it:

'use strict';

let moment = require('moment');
let ExchangeCandlestick = require('../dict/exchange_candlestick')
let _ = require('lodash')
let resample = require('./../utils/resample')

module.exports = class Backfill {
    constructor(exchangesIterator, candleImporter) {
        this.exchangesIterator = exchangesIterator
        this.candleImporter = candleImporter
    }

    async backfill(exchangeName, symbol, period, date) {
        let exchange = this.exchangesIterator.find(e => e.getName() === exchangeName)
        if (!exchange) {
            throw 'Exchange not found: ' + exchangeName
        }

        let start = moment().subtract(date, 'days');
        let candles = undefined
        let totalResults = []

        do {
            console.log('Since: ' + new Date(start).toISOString())
            candles = await exchange.backfill(symbol, period, start)
            totalResults = totalResults.concat(candles)

            let exchangeCandlesticks = candles.map(candle => {
                return ExchangeCandlestick.createFromCandle(exchangeName, symbol, period, candle)
            });

            await this.candleImporter.insertCandles(exchangeCandlesticks)

            console.log('Got: ' + candles.length + ' candles')

            start = _.max(candles.map(r => new Date(r.time * 1000)))
        } while (candles.length > 10);
        console.log('Backfill Finished');

        // @TODO This should be exported into a customizing/global config? 
        // Same as in the exchanges, e.g. bitmex.js
        let resamples = {};

        if (!resamples[symbol.symbol]) {
            resamples[symbol.symbol] = {}
        }

        if (!resamples[symbol.symbol]['5m']) {
            resamples[symbol.symbol]['5m'] = []
        }

        resamples[symbol.symbol]['5m'].push('10m')

        if (!resamples[symbol.symbol]['5m']) {
            resamples[symbol.symbol]['5m'] = []
        }

        resamples[symbol.symbol]['5m'].push('15m')

        let preparedTotalResults = totalResults.reverse().slice();
        // lets wait for settle down of database insert: per design we dont know when is was inserted to database
        if (resamples[symbol['symbol']] && resamples[symbol['symbol']][period] && resamples[symbol['symbol']][period].length > 0) {
            console.log('Start Resampling')
            setTimeout(async () => {
                for (let periodTo of resamples[symbol['symbol']][period]) {
                    let resampledCandles = resample.resampleMinutes(
                        preparedTotalResults,
                        resample.convertPeriodToMinute(periodTo)
                    )
                    let resampledCandlesticks = resampledCandles.map(candle => {
                        return ExchangeCandlestick.createFromCandle(exchangeName, symbol, periodTo, candle)
                    });
        
                    await this.candleImporter.insertCandles(resampledCandlesticks)

                    console.log('Resampling: ' + period + ' -> ' + periodTo)
                }
                console.log('Resampling Finished')

            }, 1000);
        }
    }
};