Resampling after Backfill? about crypto-trading-bot

I have implemented this in the modules/backfill.js. If you like it you could use it for the main master.

Whole file:

'use strict';

var moment = require('moment');
let CandlestickEvent = require('../event/candlestick_event')
var _ = require('lodash')
let resample = require('./../utils/resample')

module.exports = class Backfill {
    constructor(exchangesIterator, candleStickListener) {
        this.exchangesIterator = exchangesIterator
        this.candleStickListener = candleStickListener
    }

    async backfill(exchangeName, symbol, period, date) {
        let exchange = this.exchangesIterator.find(e => e.getName() === exchangeName)
        if (!exchange) {
            throw 'Exchange not found: ' + exchangeName
        }

        let start = moment().subtract(date, 'days');
        let results = undefined
        let totalResults = []

        do {
            console.log('Since: ' + start)
            results = await exchange.backfill(symbol, period, start)
            totalResults = totalResults.concat(results)

            let event = new CandlestickEvent(exchangeName, symbol, period, results)

            await this.candleStickListener.onCandleStick(event)

            console.log('Got: ' + results.length)

            start = _.max(results.map(r => new Date(r.time * 1000)))
        } while (results.length > 10);

        // @TODO This should be exported into a customizing/global config? 
        // Same as in the exchanges, e.g. bitmex.js
        let resamples = {};

        if (!resamples[symbol.symbol]) {
            resamples[symbol.symbol] = {}
        }

        if (!resamples[symbol.symbol]['5m']) {
            resamples[symbol.symbol]['5m'] = []
        }

        resamples[symbol.symbol]['5m'].push('15m')

        // lets wait for settle down of database insert: per design we dont know when is was inserted to database
        if (resamples[symbol['symbol']] && resamples[symbol['symbol']][period] && resamples[symbol['symbol']][period].length > 0) {
            console.log('Start Resampling')
            setTimeout(async () => {
                for (let periodTo of resamples[symbol['symbol']][period]) {
                    let resampledCandles = resample.resampleMinutes(
                        totalResults.reverse().slice(),
                        resample.convertPeriodToMinute(periodTo)
                    )
                    let event = new CandlestickEvent(exchangeName, symbol, periodTo, resampledCandles)

                    await this.candleStickListener.onCandleStick(event)

                    console.log('Resampling: ' + period + ' -> ' + periodTo)
                }
                console.log('Resampling Finished')

            }, 1000);
        }
    }
};

The referenced commit does not include a resampling of the backfilled data, does it?

[Haehnchen]

no, but resulted in that not all resampled data was saved

Ah, I see okay, thanks!

I have update my coding for this. Feel free to use it, if you like it:

'use strict';

let moment = require('moment');
let ExchangeCandlestick = require('../dict/exchange_candlestick')
let _ = require('lodash')
let resample = require('./../utils/resample')

module.exports = class Backfill {
    constructor(exchangesIterator, candleImporter) {
        this.exchangesIterator = exchangesIterator
        this.candleImporter = candleImporter
    }

    async backfill(exchangeName, symbol, period, date) {
        let exchange = this.exchangesIterator.find(e => e.getName() === exchangeName)
        if (!exchange) {
            throw 'Exchange not found: ' + exchangeName
        }

        let start = moment().subtract(date, 'days');
        let candles = undefined
        let totalResults = []

        do {
            console.log('Since: ' + new Date(start).toISOString())
            candles = await exchange.backfill(symbol, period, start)
            totalResults = totalResults.concat(candles)

            let exchangeCandlesticks = candles.map(candle => {
                return ExchangeCandlestick.createFromCandle(exchangeName, symbol, period, candle)
            });

            await this.candleImporter.insertCandles(exchangeCandlesticks)

            console.log('Got: ' + candles.length + ' candles')

            start = _.max(candles.map(r => new Date(r.time * 1000)))
        } while (candles.length > 10);
        console.log('Backfill Finished');

        // @TODO This should be exported into a customizing/global config? 
        // Same as in the exchanges, e.g. bitmex.js
        let resamples = {};

        if (!resamples[symbol.symbol]) {
            resamples[symbol.symbol] = {}
        }

        if (!resamples[symbol.symbol]['5m']) {
            resamples[symbol.symbol]['5m'] = []
        }

        resamples[symbol.symbol]['5m'].push('10m')

        if (!resamples[symbol.symbol]['5m']) {
            resamples[symbol.symbol]['5m'] = []
        }

        resamples[symbol.symbol]['5m'].push('15m')

        let preparedTotalResults = totalResults.reverse().slice();
        // lets wait for settle down of database insert: per design we dont know when is was inserted to database
        if (resamples[symbol['symbol']] && resamples[symbol['symbol']][period] && resamples[symbol['symbol']][period].length > 0) {
            console.log('Start Resampling')
            setTimeout(async () => {
                for (let periodTo of resamples[symbol['symbol']][period]) {
                    let resampledCandles = resample.resampleMinutes(
                        preparedTotalResults,
                        resample.convertPeriodToMinute(periodTo)
                    )
                    let resampledCandlesticks = resampledCandles.map(candle => {
                        return ExchangeCandlestick.createFromCandle(exchangeName, symbol, periodTo, candle)
                    });
        
                    await this.candleImporter.insertCandles(resampledCandlesticks)

                    console.log('Resampling: ' + period + ' -> ' + periodTo)
                }
                console.log('Resampling Finished')

            }, 1000);
        }
    }
};