diff --git a/CHANGELOG.md b/CHANGELOG.md index b1bf851a1..710d62144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ - ([\#455](https://github.com/forbole/bdjuno/pull/455)) Added `unbonding_tokens` and `staked_not_bonded_tokens` values to staking pool table +#### Daily refetch +- ([\#454](https://github.com/forbole/bdjuno/pull/454)) Added `daily refetch` module to refetch missing blocks every day + + ## Version v3.2.0 ### Changes #### Mint module diff --git a/database/daily_refetch.go b/database/daily_refetch.go new file mode 100644 index 000000000..eefb51b7e --- /dev/null +++ b/database/daily_refetch.go @@ -0,0 +1,24 @@ +package database + +// GetTotalBlocks implements database.Database +func (db *Db) GetTotalBlocks() (int64, error) { + var blockCount int64 + err := db.Sql.QueryRow(`SELECT count(*) FROM block;`).Scan(&blockCount) + return blockCount, err +} + +// GetMissingBlocks returns an array of missing blocks from one day ago +func (db *Db) GetMissingBlocks(startHeight, endHeight int64) []int64 { + var result []int64 + stmt := `SELECT generate_series($1::int,$2::int) EXCEPT SELECT height FROM block ORDER BY 1;` + err := db.Sqlx.Select(&result, stmt, startHeight, endHeight) + if err != nil { + return nil + } + + if len(result) == 0 { + return nil + } + + return result +} diff --git a/modules/daily_refetch/handle_periodic_operations.go b/modules/daily_refetch/handle_periodic_operations.go new file mode 100644 index 000000000..cdb2cce86 --- /dev/null +++ b/modules/daily_refetch/handle_periodic_operations.go @@ -0,0 +1,71 @@ +package daily_refetch + +import ( + "fmt" + "time" + + "github.com/forbole/juno/v3/parser" + "github.com/forbole/juno/v3/types/config" + + "github.com/go-co-op/gocron" + "github.com/rs/zerolog/log" + + parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types" +) + +func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { + log.Debug().Str("module", "daily refetch").Msg("setting up periodic tasks") + + // Setup a cron job to run every midnight + if _, err := scheduler.Every(1).Day().At("00:00").Do(func() { + m.refetchMissingBlocks() + }); err != nil { + return fmt.Errorf("error while setting up daily refetch periodic operation: %s", err) + } + + return nil +} + +// refetchMissingBlocks checks for missing blocks from one day ago and refetches them +func (m *Module) refetchMissingBlocks() error { + log.Trace().Str("module", "daily refetch").Str("refetching", "blocks"). + Msg("refetching missing blocks") + + latestBlock, err := m.node.LatestHeight() + if err != nil { + return fmt.Errorf("error while getting latest block: %s", err) + } + + blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now()) + if err != nil { + return fmt.Errorf("error while getting block height from a day ago: %s", err) + } + var startHeight int64 = blockHeightDayAgo.Height + + missingBlocks := m.database.GetMissingBlocks(startHeight, latestBlock) + + // return if no blocks are missing + if len(missingBlocks) == 0 { + return nil + } + + parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig()) + if err != nil { + return err + } + + workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) + worker := parser.NewWorker(workerCtx, nil, 0) + + log.Info().Int64("start height", startHeight).Int64("end height", latestBlock). + Msg("getting missing blocks and transactions from a day ago") + for _, block := range missingBlocks { + err = worker.Process(block) + if err != nil { + return fmt.Errorf("error while re-fetching block %d: %s", block, err) + } + } + + return nil + +} diff --git a/modules/daily_refetch/module.go b/modules/daily_refetch/module.go new file mode 100644 index 000000000..44fa8afb2 --- /dev/null +++ b/modules/daily_refetch/module.go @@ -0,0 +1,35 @@ +package daily_refetch + +import ( + "github.com/forbole/juno/v3/node" + + bdjunodb "github.com/forbole/bdjuno/v3/database" + + "github.com/forbole/juno/v3/modules" +) + +var ( + _ modules.Module = &Module{} + _ modules.PeriodicOperationsModule = &Module{} +) + +type Module struct { + node node.Node + database *bdjunodb.Db +} + +// NewModule builds a new Module instance +func NewModule( + node node.Node, + database *bdjunodb.Db, +) *Module { + return &Module{ + node: node, + database: database, + } +} + +// Name implements modules.Module +func (m *Module) Name() string { + return "daily refetch" +} diff --git a/modules/registrar.go b/modules/registrar.go index 90b77b7bd..c1d753f78 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -24,6 +24,7 @@ import ( "github.com/forbole/bdjuno/v3/modules/distribution" "github.com/forbole/bdjuno/v3/modules/feegrant" + dailyrefetch "github.com/forbole/bdjuno/v3/modules/daily_refetch" "github.com/forbole/bdjuno/v3/modules/gov" "github.com/forbole/bdjuno/v3/modules/mint" "github.com/forbole/bdjuno/v3/modules/modules" @@ -75,6 +76,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { authModule := auth.NewModule(r.parser, cdc, db) bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db) consensusModule := consensus.NewModule(db) + dailyRefetchModule := dailyrefetch.NewModule(ctx.Proxy, db) distrModule := distribution.NewModule(sources.DistrSource, cdc, db) feegrantModule := feegrant.NewModule(cdc, db) mintModule := mint.NewModule(sources.MintSource, cdc, db) @@ -91,6 +93,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { authModule, bankModule, consensusModule, + dailyRefetchModule, distrModule, feegrantModule, govModule,