From 63367de4dfa45b5043782b55461bcd5aee4dcf32 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Tue, 16 Aug 2022 23:14:12 +0700 Subject: [PATCH 01/17] add block module --- modules/block/module.go | 35 +++++++++++++++++++++++++++++++++++ modules/registrar.go | 3 +++ 2 files changed, 38 insertions(+) create mode 100644 modules/block/module.go diff --git a/modules/block/module.go b/modules/block/module.go new file mode 100644 index 000000000..1f080f905 --- /dev/null +++ b/modules/block/module.go @@ -0,0 +1,35 @@ +package block + +import ( + // "github.com/forbole/juno/v3/node" + + bdjunodb "github.com/forbole/bdjuno/v3/database" + + "github.com/forbole/juno/v3/modules" +) + +var ( + _ modules.Module = &Module{} + _ modules.PeriodicOperationsModule = &Module{} +) + +type Module struct { + // node node.Node + database *bdjunodb.Db +} + +// NewModule builds a new Module instance +func NewModule( + // node node.Node, + database *bdjunodb.Db, +) *Module { + return &Module{ + // node: node, + database: database, + } +} + +// Name implements modules.Module +func (m *Module) Name() string { + return "block" +} diff --git a/modules/registrar.go b/modules/registrar.go index 90b77b7bd..0503889a6 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -24,6 +24,7 @@ import ( "github.com/forbole/bdjuno/v3/modules/distribution" "github.com/forbole/bdjuno/v3/modules/feegrant" + "github.com/forbole/bdjuno/v3/modules/block" "github.com/forbole/bdjuno/v3/modules/gov" "github.com/forbole/bdjuno/v3/modules/mint" "github.com/forbole/bdjuno/v3/modules/modules" @@ -74,6 +75,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { actionsModule := actions.NewModule(ctx.JunoConfig, ctx.EncodingConfig) authModule := auth.NewModule(r.parser, cdc, db) bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db) + blockModule := block.NewModule(db) consensusModule := consensus.NewModule(db) distrModule := distribution.NewModule(sources.DistrSource, cdc, db) feegrantModule := feegrant.NewModule(cdc, db) @@ -90,6 +92,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { actionsModule, authModule, bankModule, + blockModule, consensusModule, distrModule, feegrantModule, From 78618df2e29c47ee5e5685129c4f79d3ce833fe2 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Tue, 16 Aug 2022 23:15:14 +0700 Subject: [PATCH 02/17] get total blocks --- database/block.go | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 database/block.go diff --git a/database/block.go b/database/block.go new file mode 100644 index 000000000..bdaf62892 --- /dev/null +++ b/database/block.go @@ -0,0 +1,8 @@ +package database + +// GetTotalBlocks implements database.Database +func (db *Db) GetTotalBlocks() (int64, error) { + var blockCount int64 + err := db.Sql.QueryRow(`SELECT count(*) FROM block;`).Scan(&blockCount) + return blockCount, err +} From 21227497671f758540d3759c2977f22fd9eebae3 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Tue, 16 Aug 2022 23:22:36 +0700 Subject: [PATCH 03/17] add checkMissingBlocks --- modules/block/handle_periodic_operations.go | 73 +++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 modules/block/handle_periodic_operations.go diff --git a/modules/block/handle_periodic_operations.go b/modules/block/handle_periodic_operations.go new file mode 100644 index 000000000..a3df3ae2d --- /dev/null +++ b/modules/block/handle_periodic_operations.go @@ -0,0 +1,73 @@ +package block + +import ( + "fmt" + + "github.com/forbole/juno/v3/parser" + + "github.com/go-co-op/gocron" + "github.com/rs/zerolog/log" + + parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types" + "github.com/forbole/juno/v3/types/config" +) + +func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { + log.Debug().Str("module", "block refetch").Msg("setting up periodic tasks") + + if _, err := scheduler.Every(2).Minutes().Do(func() { + m.checkMissingBlocks() + }); err != nil { + return fmt.Errorf("error while setting up block refetch periodic operation: %s", err) + } + + return nil +} + +// checkMissingBlocks checks for any missing blocks and refetches it +func (m *Module) checkMissingBlocks() error { + log.Trace().Str("module", "blocks").Str("refetching", "blocks"). + Msg("refetching missing blocks") + + // latestBlock, err := m.node.LatestHeight() + // if err != nil { + // return fmt.Errorf("error while getting latest block: %s", err) + // } + + // blockCount, err := m.database.GetTotalBlocks() + // if err != nil { + // return fmt.Errorf("error while getting last block: %s", err) + // } + + // averageBlockPerDay, err := m.db.GetAverageBlockPerDay() + // if err != nil { + // return fmt.Errorf("error while getting average day block time: %s", err) + // } + + var startHeight int64 = 1 + + var endHeight int64 = 123911 + + // if blockCount != latestBlock { + parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, nil) + if err != nil { + return err + } + + workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) + worker := parser.NewWorker(workerCtx, nil, 0) + + log.Info().Int64("start height", startHeight).Int64("end height", endHeight). + Msg("getting missing blocks and transactions") + for k := startHeight; k <= endHeight; k++ { + err = worker.ProcessIfNotExists(k) + if err != nil { + return fmt.Errorf("error while re-fetching block %d: %s", k, err) + } + } + + // } + + return nil + +} From 9a931335a15dccd2b05828122c43de0b0e17b2e4 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Tue, 16 Aug 2022 23:35:54 +0700 Subject: [PATCH 04/17] updated block module --- modules/block/handle_periodic_operations.go | 46 ++++++++++----------- modules/block/module.go | 13 ++++-- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/modules/block/handle_periodic_operations.go b/modules/block/handle_periodic_operations.go index a3df3ae2d..06ec7b4a4 100644 --- a/modules/block/handle_periodic_operations.go +++ b/modules/block/handle_periodic_operations.go @@ -29,15 +29,15 @@ func (m *Module) checkMissingBlocks() error { log.Trace().Str("module", "blocks").Str("refetching", "blocks"). Msg("refetching missing blocks") - // latestBlock, err := m.node.LatestHeight() - // if err != nil { - // return fmt.Errorf("error while getting latest block: %s", err) - // } + latestBlock, err := m.node.LatestHeight() + if err != nil { + return fmt.Errorf("error while getting latest block: %s", err) + } - // blockCount, err := m.database.GetTotalBlocks() - // if err != nil { - // return fmt.Errorf("error while getting last block: %s", err) - // } + blockCount, err := m.database.GetTotalBlocks() + if err != nil { + return fmt.Errorf("error while getting last block: %s", err) + } // averageBlockPerDay, err := m.db.GetAverageBlockPerDay() // if err != nil { @@ -48,25 +48,25 @@ func (m *Module) checkMissingBlocks() error { var endHeight int64 = 123911 - // if blockCount != latestBlock { - parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, nil) - if err != nil { - return err - } + if blockCount != latestBlock { + parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, m.config) + if err != nil { + return err + } - workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) - worker := parser.NewWorker(workerCtx, nil, 0) + workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) + worker := parser.NewWorker(workerCtx, nil, 0) - log.Info().Int64("start height", startHeight).Int64("end height", endHeight). - Msg("getting missing blocks and transactions") - for k := startHeight; k <= endHeight; k++ { - err = worker.ProcessIfNotExists(k) - if err != nil { - return fmt.Errorf("error while re-fetching block %d: %s", k, err) + log.Info().Int64("start height", startHeight).Int64("end height", endHeight). + Msg("getting missing blocks and transactions") + for k := startHeight; k <= endHeight; k++ { + err = worker.ProcessIfNotExists(k) + if err != nil { + return fmt.Errorf("error while re-fetching block %d: %s", k, err) + } } - } - // } + } return nil diff --git a/modules/block/module.go b/modules/block/module.go index 1f080f905..16f999307 100644 --- a/modules/block/module.go +++ b/modules/block/module.go @@ -1,9 +1,10 @@ package block import ( - // "github.com/forbole/juno/v3/node" + "github.com/forbole/juno/v3/node" bdjunodb "github.com/forbole/bdjuno/v3/database" + parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types" "github.com/forbole/juno/v3/modules" ) @@ -14,17 +15,21 @@ var ( ) type Module struct { - // node node.Node + node node.Node database *bdjunodb.Db + config *parsecmdtypes.Config + // config.Config } // NewModule builds a new Module instance func NewModule( - // node node.Node, + node node.Node, + config *parsecmdtypes.Config, database *bdjunodb.Db, ) *Module { return &Module{ - // node: node, + node: node, + config: config, database: database, } } From 95669d336a06a4814862360c56fc1abb72fc29d7 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 18:33:43 +0700 Subject: [PATCH 05/17] update block module and periodic ops --- modules/block/handle_periodic_operations.go | 4 ++-- modules/block/module.go | 9 ++++----- modules/registrar.go | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/block/handle_periodic_operations.go b/modules/block/handle_periodic_operations.go index 06ec7b4a4..0879450d4 100644 --- a/modules/block/handle_periodic_operations.go +++ b/modules/block/handle_periodic_operations.go @@ -4,12 +4,12 @@ import ( "fmt" "github.com/forbole/juno/v3/parser" + "github.com/forbole/juno/v3/types/config" "github.com/go-co-op/gocron" "github.com/rs/zerolog/log" parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types" - "github.com/forbole/juno/v3/types/config" ) func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { @@ -49,7 +49,7 @@ func (m *Module) checkMissingBlocks() error { var endHeight int64 = 123911 if blockCount != latestBlock { - parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, m.config) + parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig()) if err != nil { return err } diff --git a/modules/block/module.go b/modules/block/module.go index 16f999307..afe576010 100644 --- a/modules/block/module.go +++ b/modules/block/module.go @@ -4,7 +4,6 @@ import ( "github.com/forbole/juno/v3/node" bdjunodb "github.com/forbole/bdjuno/v3/database" - parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types" "github.com/forbole/juno/v3/modules" ) @@ -17,19 +16,19 @@ var ( type Module struct { node node.Node database *bdjunodb.Db - config *parsecmdtypes.Config + // config parsecmdtypes.Config // config.Config } // NewModule builds a new Module instance func NewModule( node node.Node, - config *parsecmdtypes.Config, + // config parsecmdtypes.Config, database *bdjunodb.Db, ) *Module { return &Module{ - node: node, - config: config, + node: node, + // config: config, database: database, } } diff --git a/modules/registrar.go b/modules/registrar.go index 0503889a6..7ecf4005e 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -75,7 +75,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { actionsModule := actions.NewModule(ctx.JunoConfig, ctx.EncodingConfig) authModule := auth.NewModule(r.parser, cdc, db) bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db) - blockModule := block.NewModule(db) + blockModule := block.NewModule(ctx.Proxy, db) consensusModule := consensus.NewModule(db) distrModule := distribution.NewModule(sources.DistrSource, cdc, db) feegrantModule := feegrant.NewModule(cdc, db) From b88baa1c7dcfe3318359fc1faa34612e64e188cd Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 19:05:15 +0700 Subject: [PATCH 06/17] update module struct --- modules/block/module.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/block/module.go b/modules/block/module.go index afe576010..a39cd1eee 100644 --- a/modules/block/module.go +++ b/modules/block/module.go @@ -16,19 +16,15 @@ var ( type Module struct { node node.Node database *bdjunodb.Db - // config parsecmdtypes.Config - // config.Config } // NewModule builds a new Module instance func NewModule( node node.Node, - // config parsecmdtypes.Config, database *bdjunodb.Db, ) *Module { return &Module{ - node: node, - // config: config, + node: node, database: database, } } From 8de3bce05c554ca1eb246781037d5bc23d392323 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 19:08:08 +0700 Subject: [PATCH 07/17] updated periodic ops --- modules/block/handle_periodic_operations.go | 24 ++++++++++----------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/modules/block/handle_periodic_operations.go b/modules/block/handle_periodic_operations.go index 0879450d4..6868e1134 100644 --- a/modules/block/handle_periodic_operations.go +++ b/modules/block/handle_periodic_operations.go @@ -2,6 +2,7 @@ package block import ( "fmt" + "time" "github.com/forbole/juno/v3/parser" "github.com/forbole/juno/v3/types/config" @@ -24,7 +25,7 @@ func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { return nil } -// checkMissingBlocks checks for any missing blocks and refetches it +// checkMissingBlocks checks for any missing blocks from one day ago and refetches it func (m *Module) checkMissingBlocks() error { log.Trace().Str("module", "blocks").Str("refetching", "blocks"). Msg("refetching missing blocks") @@ -36,17 +37,14 @@ func (m *Module) checkMissingBlocks() error { blockCount, err := m.database.GetTotalBlocks() if err != nil { - return fmt.Errorf("error while getting last block: %s", err) + return fmt.Errorf("error while getting block count: %s", err) } - // averageBlockPerDay, err := m.db.GetAverageBlockPerDay() - // if err != nil { - // return fmt.Errorf("error while getting average day block time: %s", err) - // } - - var startHeight int64 = 1 - - var endHeight int64 = 123911 + blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now()) + if err != nil { + return fmt.Errorf("error while getting block height from a day ago: %s", err) + } + var startHeight int64 = blockHeightDayAgo.Height if blockCount != latestBlock { parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig()) @@ -57,9 +55,9 @@ func (m *Module) checkMissingBlocks() error { workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) worker := parser.NewWorker(workerCtx, nil, 0) - log.Info().Int64("start height", startHeight).Int64("end height", endHeight). - Msg("getting missing blocks and transactions") - for k := startHeight; k <= endHeight; k++ { + log.Info().Int64("start height", startHeight).Int64("end height", latestBlock). + Msg("getting missing blocks and transactions from a day ago") + for k := startHeight; k <= latestBlock; k++ { err = worker.ProcessIfNotExists(k) if err != nil { return fmt.Errorf("error while re-fetching block %d: %s", k, err) From 7291f58e2dd0d3d43770e486aedda97bef33b307 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 20:07:38 +0700 Subject: [PATCH 08/17] rename to daily refetch --- .../handle_periodic_operations.go | 14 +++++++------- modules/{block => daily_refetch}/module.go | 4 ++-- modules/registrar.go | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) rename modules/{block => daily_refetch}/handle_periodic_operations.go (81%) rename modules/{block => daily_refetch}/module.go (92%) diff --git a/modules/block/handle_periodic_operations.go b/modules/daily_refetch/handle_periodic_operations.go similarity index 81% rename from modules/block/handle_periodic_operations.go rename to modules/daily_refetch/handle_periodic_operations.go index 6868e1134..c07ba4f21 100644 --- a/modules/block/handle_periodic_operations.go +++ b/modules/daily_refetch/handle_periodic_operations.go @@ -1,4 +1,4 @@ -package block +package daily_refetch import ( "fmt" @@ -14,20 +14,20 @@ import ( ) func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { - log.Debug().Str("module", "block refetch").Msg("setting up periodic tasks") + log.Debug().Str("module", "daily refetch").Msg("setting up periodic tasks") if _, err := scheduler.Every(2).Minutes().Do(func() { - m.checkMissingBlocks() + m.refetchMissingBlocks() }); err != nil { - return fmt.Errorf("error while setting up block refetch periodic operation: %s", err) + return fmt.Errorf("error while setting up daily refetch periodic operation: %s", err) } return nil } -// checkMissingBlocks checks for any missing blocks from one day ago and refetches it -func (m *Module) checkMissingBlocks() error { - log.Trace().Str("module", "blocks").Str("refetching", "blocks"). +// refetchMissingBlocks checks for missing blocks from one day ago and refetches them +func (m *Module) refetchMissingBlocks() error { + log.Trace().Str("module", "daily refetch").Str("refetching", "blocks"). Msg("refetching missing blocks") latestBlock, err := m.node.LatestHeight() diff --git a/modules/block/module.go b/modules/daily_refetch/module.go similarity index 92% rename from modules/block/module.go rename to modules/daily_refetch/module.go index a39cd1eee..44fa8afb2 100644 --- a/modules/block/module.go +++ b/modules/daily_refetch/module.go @@ -1,4 +1,4 @@ -package block +package daily_refetch import ( "github.com/forbole/juno/v3/node" @@ -31,5 +31,5 @@ func NewModule( // Name implements modules.Module func (m *Module) Name() string { - return "block" + return "daily refetch" } diff --git a/modules/registrar.go b/modules/registrar.go index 7ecf4005e..d47e0e781 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -24,7 +24,7 @@ import ( "github.com/forbole/bdjuno/v3/modules/distribution" "github.com/forbole/bdjuno/v3/modules/feegrant" - "github.com/forbole/bdjuno/v3/modules/block" + "github.com/forbole/bdjuno/v3/modules/daily_refetch" "github.com/forbole/bdjuno/v3/modules/gov" "github.com/forbole/bdjuno/v3/modules/mint" "github.com/forbole/bdjuno/v3/modules/modules" @@ -75,8 +75,8 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { actionsModule := actions.NewModule(ctx.JunoConfig, ctx.EncodingConfig) authModule := auth.NewModule(r.parser, cdc, db) bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db) - blockModule := block.NewModule(ctx.Proxy, db) consensusModule := consensus.NewModule(db) + dailyRefetchModule := daily_refetch.NewModule(ctx.Proxy, db) distrModule := distribution.NewModule(sources.DistrSource, cdc, db) feegrantModule := feegrant.NewModule(cdc, db) mintModule := mint.NewModule(sources.MintSource, cdc, db) @@ -92,8 +92,8 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { actionsModule, authModule, bankModule, - blockModule, consensusModule, + dailyRefetchModule, distrModule, feegrantModule, govModule, From 1fdf3d8ae032818afa1c620d3ca7209561f9b149 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 20:13:05 +0700 Subject: [PATCH 09/17] updated registrar.go --- modules/registrar.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/registrar.go b/modules/registrar.go index d47e0e781..c1d753f78 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -24,7 +24,7 @@ import ( "github.com/forbole/bdjuno/v3/modules/distribution" "github.com/forbole/bdjuno/v3/modules/feegrant" - "github.com/forbole/bdjuno/v3/modules/daily_refetch" + dailyrefetch "github.com/forbole/bdjuno/v3/modules/daily_refetch" "github.com/forbole/bdjuno/v3/modules/gov" "github.com/forbole/bdjuno/v3/modules/mint" "github.com/forbole/bdjuno/v3/modules/modules" @@ -76,7 +76,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { authModule := auth.NewModule(r.parser, cdc, db) bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db) consensusModule := consensus.NewModule(db) - dailyRefetchModule := daily_refetch.NewModule(ctx.Proxy, db) + dailyRefetchModule := dailyrefetch.NewModule(ctx.Proxy, db) distrModule := distribution.NewModule(sources.DistrSource, cdc, db) feegrantModule := feegrant.NewModule(cdc, db) mintModule := mint.NewModule(sources.MintSource, cdc, db) From 023ae520a7eb582e4d0b569cac007b5b36eaee61 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 20:45:54 +0700 Subject: [PATCH 10/17] updated scheduler --- modules/daily_refetch/handle_periodic_operations.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/daily_refetch/handle_periodic_operations.go b/modules/daily_refetch/handle_periodic_operations.go index c07ba4f21..2ff5bb9fc 100644 --- a/modules/daily_refetch/handle_periodic_operations.go +++ b/modules/daily_refetch/handle_periodic_operations.go @@ -16,7 +16,8 @@ import ( func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { log.Debug().Str("module", "daily refetch").Msg("setting up periodic tasks") - if _, err := scheduler.Every(2).Minutes().Do(func() { + // Setup a cron job to run every midnight + if _, err := scheduler.Every(1).Day().At("00:00").Do(func() { m.refetchMissingBlocks() }); err != nil { return fmt.Errorf("error while setting up daily refetch periodic operation: %s", err) @@ -25,7 +26,7 @@ func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error { return nil } -// refetchMissingBlocks checks for missing blocks from one day ago and refetches them +// refetchMissingBlocks checks for missing blocks from one day ago and refetches them func (m *Module) refetchMissingBlocks() error { log.Trace().Str("module", "daily refetch").Str("refetching", "blocks"). Msg("refetching missing blocks") From af1a8223133d3a6b8f09cac1de5300feedb64f4a Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 20:47:44 +0700 Subject: [PATCH 11/17] updated database --- database/{block.go => daily_refetch.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename database/{block.go => daily_refetch.go} (100%) diff --git a/database/block.go b/database/daily_refetch.go similarity index 100% rename from database/block.go rename to database/daily_refetch.go From 66285742f4962ea2e900ba78d6b875458feebbf1 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Wed, 17 Aug 2022 20:53:21 +0700 Subject: [PATCH 12/17] added CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce15182fb..c1729f5b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ #### Staking Module - ([\#443](https://github.com/forbole/bdjuno/pull/443)) Remove tombstone status from staking module(already stored in slashing module) +#### Daily refetch +- ([\#454](https://github.com/forbole/bdjuno/pull/454)) Added `daily refetch` module to refetch missing blocks every day + + ## Version v3.2.0 ### Changes #### Mint module From 98e340ef975a2e0ac1b2e73ef550f73278dac4d8 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Thu, 18 Aug 2022 18:45:59 +0700 Subject: [PATCH 13/17] updated periodic ops --- .../handle_periodic_operations.go | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/modules/daily_refetch/handle_periodic_operations.go b/modules/daily_refetch/handle_periodic_operations.go index 2ff5bb9fc..a5d708d52 100644 --- a/modules/daily_refetch/handle_periodic_operations.go +++ b/modules/daily_refetch/handle_periodic_operations.go @@ -41,30 +41,32 @@ func (m *Module) refetchMissingBlocks() error { return fmt.Errorf("error while getting block count: %s", err) } + if blockCount >= latestBlock { + return nil + } + + parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig()) + if err != nil { + return err + } + + workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) + worker := parser.NewWorker(workerCtx, nil, 0) + blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now()) if err != nil { return fmt.Errorf("error while getting block height from a day ago: %s", err) } + var startHeight int64 = blockHeightDayAgo.Height - if blockCount != latestBlock { - parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig()) + log.Info().Int64("start height", startHeight).Int64("end height", latestBlock). + Msg("getting missing blocks and transactions from a day ago") + for k := startHeight; k <= latestBlock; k++ { + err = worker.ProcessIfNotExists(k) if err != nil { - return err + return fmt.Errorf("error while re-fetching block %d: %s", k, err) } - - workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) - worker := parser.NewWorker(workerCtx, nil, 0) - - log.Info().Int64("start height", startHeight).Int64("end height", latestBlock). - Msg("getting missing blocks and transactions from a day ago") - for k := startHeight; k <= latestBlock; k++ { - err = worker.ProcessIfNotExists(k) - if err != nil { - return fmt.Errorf("error while re-fetching block %d: %s", k, err) - } - } - } return nil From 128371378359fe09e707ecbd494ea37aee7827fe Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Thu, 25 Aug 2022 15:25:31 +0700 Subject: [PATCH 14/17] add GetMissingBlocks func --- database/daily_refetch.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/database/daily_refetch.go b/database/daily_refetch.go index bdaf62892..b087e411d 100644 --- a/database/daily_refetch.go +++ b/database/daily_refetch.go @@ -6,3 +6,19 @@ func (db *Db) GetTotalBlocks() (int64, error) { err := db.Sql.QueryRow(`SELECT count(*) FROM block;`).Scan(&blockCount) return blockCount, err } + +// GetMissingBlocks returns an array of missing blocks from one day ago +func (db *Db) GetMissingBlocks(startHeight, endHeight int64) []int64 { + var result []int64 + stmt := `SELECT generate_series($1 , $2) EXCEPT SELECT height FROM block ORDER BY 1;` + err := db.Sqlx.Select(&result, stmt, startHeight, endHeight) + if err != nil { + return nil + } + + if len(result) == 0 { + return nil + } + + return result +} From 3ee304570eb41d6baaa96e75acf7c9041828539f Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Thu, 25 Aug 2022 15:38:40 +0700 Subject: [PATCH 15/17] updated periodic ops --- .../handle_periodic_operations.go | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/modules/daily_refetch/handle_periodic_operations.go b/modules/daily_refetch/handle_periodic_operations.go index a5d708d52..fc6df7099 100644 --- a/modules/daily_refetch/handle_periodic_operations.go +++ b/modules/daily_refetch/handle_periodic_operations.go @@ -36,12 +36,16 @@ func (m *Module) refetchMissingBlocks() error { return fmt.Errorf("error while getting latest block: %s", err) } - blockCount, err := m.database.GetTotalBlocks() + blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now()) if err != nil { - return fmt.Errorf("error while getting block count: %s", err) + return fmt.Errorf("error while getting block height from a day ago: %s", err) } + var startHeight int64 = blockHeightDayAgo.Height - if blockCount >= latestBlock { + missingBlocks := m.database.GetMissingBlocks(startHeight, latestBlock) + + // return if no blocks are missing + if len(missingBlocks) == 0 { return nil } @@ -53,19 +57,12 @@ func (m *Module) refetchMissingBlocks() error { workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules) worker := parser.NewWorker(workerCtx, nil, 0) - blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now()) - if err != nil { - return fmt.Errorf("error while getting block height from a day ago: %s", err) - } - - var startHeight int64 = blockHeightDayAgo.Height - log.Info().Int64("start height", startHeight).Int64("end height", latestBlock). Msg("getting missing blocks and transactions from a day ago") - for k := startHeight; k <= latestBlock; k++ { - err = worker.ProcessIfNotExists(k) + for _, block := range missingBlocks { + err = worker.ProcessIfNotExists(block) if err != nil { - return fmt.Errorf("error while re-fetching block %d: %s", k, err) + return fmt.Errorf("error while re-fetching block %d: %s", block, err) } } From d6ea458419cb50bf9f87856a1e227e429c048059 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Thu, 25 Aug 2022 16:55:13 +0700 Subject: [PATCH 16/17] updated sql stmt --- database/daily_refetch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/daily_refetch.go b/database/daily_refetch.go index b087e411d..eefb51b7e 100644 --- a/database/daily_refetch.go +++ b/database/daily_refetch.go @@ -10,7 +10,7 @@ func (db *Db) GetTotalBlocks() (int64, error) { // GetMissingBlocks returns an array of missing blocks from one day ago func (db *Db) GetMissingBlocks(startHeight, endHeight int64) []int64 { var result []int64 - stmt := `SELECT generate_series($1 , $2) EXCEPT SELECT height FROM block ORDER BY 1;` + stmt := `SELECT generate_series($1::int,$2::int) EXCEPT SELECT height FROM block ORDER BY 1;` err := db.Sqlx.Select(&result, stmt, startHeight, endHeight) if err != nil { return nil From f22401af63b259edfb985abed5074a612802bc8d Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Thu, 25 Aug 2022 17:02:01 +0700 Subject: [PATCH 17/17] use Process func --- modules/daily_refetch/handle_periodic_operations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/daily_refetch/handle_periodic_operations.go b/modules/daily_refetch/handle_periodic_operations.go index fc6df7099..cdb2cce86 100644 --- a/modules/daily_refetch/handle_periodic_operations.go +++ b/modules/daily_refetch/handle_periodic_operations.go @@ -60,7 +60,7 @@ func (m *Module) refetchMissingBlocks() error { log.Info().Int64("start height", startHeight).Int64("end height", latestBlock). Msg("getting missing blocks and transactions from a day ago") for _, block := range missingBlocks { - err = worker.ProcessIfNotExists(block) + err = worker.Process(block) if err != nil { return fmt.Errorf("error while re-fetching block %d: %s", block, err) }