From de1672cda63d2748378435aef7ba0e9664a2121b Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Tue, 27 Dec 2022 18:20:26 +0800 Subject: [PATCH 01/15] add parse cmd to top accounts module with workers --- cmd/parse/parse.go | 2 + cmd/parse/top_accounts/all.go | 121 ++++++++++++++++++ cmd/parse/top_accounts/cmd.go | 18 +++ cmd/parse/top_accounts/worker.go | 86 +++++++++++++ modules/auth/module.go | 5 +- modules/auth/source/local/source.go | 33 +++++ modules/auth/source/remote/source.go | 69 ++++++++++ modules/auth/source/source.go | 7 + .../auth/utils_refresh_all_base_accounts.go | 42 ++++++ modules/registrar.go | 2 +- modules/top_accounts/handle_msg.go | 10 +- .../handle_periodic_operations.go | 2 +- modules/top_accounts/utils_refresh.go | 4 +- modules/types/sources.go | 7 + 14 files changed, 398 insertions(+), 10 deletions(-) create mode 100644 cmd/parse/top_accounts/all.go create mode 100644 cmd/parse/top_accounts/cmd.go create mode 100644 cmd/parse/top_accounts/worker.go create mode 100644 modules/auth/source/local/source.go create mode 100644 modules/auth/source/remote/source.go create mode 100644 modules/auth/source/source.go create mode 100644 modules/auth/utils_refresh_all_base_accounts.go diff --git a/cmd/parse/parse.go b/cmd/parse/parse.go index 34061227d..fcad60e76 100644 --- a/cmd/parse/parse.go +++ b/cmd/parse/parse.go @@ -16,6 +16,7 @@ import ( parsemint "github.com/forbole/bdjuno/v3/cmd/parse/mint" parsepricefeed "github.com/forbole/bdjuno/v3/cmd/parse/pricefeed" parsestaking "github.com/forbole/bdjuno/v3/cmd/parse/staking" + parsetopaccounts "github.com/forbole/bdjuno/v3/cmd/parse/top_accounts" parsetransaction "github.com/forbole/juno/v3/cmd/parse/transactions" ) @@ -38,6 +39,7 @@ func NewParseCmd(parseCfg *parse.Config) *cobra.Command { parsemint.NewMintCmd(parseCfg), parsepricefeed.NewPricefeedCmd(parseCfg), parsestaking.NewStakingCmd(parseCfg), + parsetopaccounts.NewTopAccountsCmd(parseCfg), parsetransaction.NewTransactionsCmd(parseCfg), ) diff --git a/cmd/parse/top_accounts/all.go b/cmd/parse/top_accounts/all.go new file mode 100644 index 000000000..1946ea888 --- /dev/null +++ b/cmd/parse/top_accounts/all.go @@ -0,0 +1,121 @@ +package top_accounts + +import ( + "fmt" + "os" + "os/signal" + "sync" + "syscall" + + "github.com/forbole/bdjuno/v3/modules/bank" + "github.com/forbole/bdjuno/v3/modules/distribution" + "github.com/forbole/bdjuno/v3/modules/staking" + topaccounts "github.com/forbole/bdjuno/v3/modules/top_accounts" + modulestypes "github.com/forbole/bdjuno/v3/modules/types" + "github.com/forbole/bdjuno/v3/types" + "github.com/rs/zerolog/log" + + parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types" + "github.com/forbole/juno/v3/parser" + "github.com/forbole/juno/v3/types/config" + "github.com/spf13/cobra" + + "github.com/forbole/bdjuno/v3/database" + "github.com/forbole/bdjuno/v3/modules/auth" +) + +var ( + waitGroup sync.WaitGroup +) + +const ( + flagWorker = "worker" +) + +func allCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { + cmd := &cobra.Command{ + Use: "all", + RunE: func(cmd *cobra.Command, args []string) error { + parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parseConfig) + if err != nil { + return err + } + + sources, err := modulestypes.BuildSources(config.Cfg.Node, parseCtx.EncodingConfig) + if err != nil { + return err + } + + // Get the database + db := database.Cast(parseCtx.Database) + + // Build modules + authModule := auth.NewModule(sources.AuthSource, nil, parseCtx.EncodingConfig.Marshaler, db) + bankModule := bank.NewModule(nil, sources.BankSource, parseCtx.EncodingConfig.Marshaler, db) + distiModule := distribution.NewModule(sources.DistrSource, parseCtx.EncodingConfig.Marshaler, db) + stakingModule := staking.NewModule(sources.StakingSource, parseCtx.EncodingConfig.Marshaler, db) + topaccountsModule := topaccounts.NewModule(nil, nil, nil, nil, parseCtx.EncodingConfig.Marshaler, db) + + // Get workers + exportQueue := NewQueue(5) + workerCount, _ := cmd.Flags().GetInt64(flagWorker) + workers := make([]Worker, workerCount) + for i := range workers { + workers[i] = NewWorker(exportQueue, bankModule, distiModule, stakingModule, topaccountsModule) + } + + waitGroup.Add(1) + + // Get all base accounts + accounts, err := authModule.GetAllBaseAccounts(0) + if err != nil { + return fmt.Errorf("error while getting base accounts: %s", err) + } + + // Store accounts + err = db.SaveAccounts(accounts) + if err != nil { + return err + } + + for i, w := range workers { + log.Debug().Int("number", i+1).Msg("starting worker...") + go w.start() + } + + trapSignal(parseCtx) + + go enqueueAddresses(exportQueue, accounts) + + waitGroup.Wait() + return nil + }, + } + + cmd.Flags().Int64(flagWorker, 1, "worker count") + + return cmd +} + +func enqueueAddresses(exportQueue AddressQueue, accounts []types.Account) { + for _, account := range accounts { + exportQueue <- account.Address + } +} + +// trapSignal will listen for any OS signal and invoke Done on the main +// WaitGroup allowing the main process to gracefully exit. +func trapSignal(ctx *parser.Context) { + var sigCh = make(chan os.Signal, 1) + + signal.Notify(sigCh, syscall.SIGTERM) + signal.Notify(sigCh, syscall.SIGINT) + + go func() { + sig := <-sigCh + log.Info().Str("signal", sig.String()).Msg("caught signal; shutting down...") + defer ctx.Node.Stop() + defer ctx.Database.Close() + defer waitGroup.Done() + }() +} diff --git a/cmd/parse/top_accounts/cmd.go b/cmd/parse/top_accounts/cmd.go new file mode 100644 index 000000000..3dd1e708c --- /dev/null +++ b/cmd/parse/top_accounts/cmd.go @@ -0,0 +1,18 @@ +package top_accounts + +import ( + parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types" + "github.com/spf13/cobra" +) + +func NewTopAccountsCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { + cmd := &cobra.Command{ + Use: "top-accounts", + } + + cmd.AddCommand( + allCmd(parseConfig), + ) + + return cmd +} diff --git a/cmd/parse/top_accounts/worker.go b/cmd/parse/top_accounts/worker.go new file mode 100644 index 000000000..eca974078 --- /dev/null +++ b/cmd/parse/top_accounts/worker.go @@ -0,0 +1,86 @@ +package top_accounts + +import ( + "fmt" + + "github.com/forbole/bdjuno/v3/modules/bank" + "github.com/forbole/bdjuno/v3/modules/distribution" + "github.com/forbole/bdjuno/v3/modules/staking" + topaccounts "github.com/forbole/bdjuno/v3/modules/top_accounts" + "github.com/rs/zerolog/log" +) + +type AddressQueue chan string + +func NewQueue(size int) AddressQueue { + return make(chan string, size) +} + +type Worker struct { + queue AddressQueue + bankModule *bank.Module + distriModule *distribution.Module + stakingModule *staking.Module + topaccountsModule *topaccounts.Module +} + +func NewWorker( + queue AddressQueue, + bankModule *bank.Module, distriModule *distribution.Module, + stakingModule *staking.Module, topaccountsModule *topaccounts.Module, +) Worker { + return Worker{ + queue: queue, + bankModule: bankModule, + distriModule: distriModule, + stakingModule: stakingModule, + topaccountsModule: topaccountsModule, + } +} + +func (w Worker) start() { + for address := range w.queue { + err := w.refreshAll(address) + if err != nil { + log.Error().Str("account", address).Msg("re-enqueueing failed address") + + go func(address string) { + w.queue <- address + }(address) + } + } +} + +func (w *Worker) refreshAll(address string) error { + err := w.bankModule.UpdateBalances([]string{address}, 0) + if err != nil { + return fmt.Errorf("error while refreshing account balance of account %s", address) + } + + err = w.stakingModule.RefreshDelegations(0, address) + if err != nil { + return fmt.Errorf("error while refreshing delegations of account %s", address) + } + + err = w.stakingModule.RefreshRedelegations(0, address) + if err != nil { + return fmt.Errorf("error while refreshing redelegations of account %s", address) + } + + err = w.stakingModule.RefreshUnbondings(0, address) + if err != nil { + return fmt.Errorf("error while refreshing unbonding delegations of account %s", address) + } + + err = w.distriModule.RefreshDelegatorRewards(0, []string{address}) + if err != nil { + return fmt.Errorf("error while refreshing rewards of account %s", address) + } + + err = w.topaccountsModule.RefreshTopAccountsSum([]string{address}) + if err != nil { + return fmt.Errorf("error while refreshing top account sum of account %s", address) + } + + return nil +} diff --git a/modules/auth/module.go b/modules/auth/module.go index 2d725cb2a..d04d2c83c 100644 --- a/modules/auth/module.go +++ b/modules/auth/module.go @@ -4,6 +4,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec" "github.com/forbole/bdjuno/v3/database" + "github.com/forbole/bdjuno/v3/modules/auth/source" "github.com/forbole/juno/v3/modules" "github.com/forbole/juno/v3/modules/messages" @@ -20,12 +21,14 @@ type Module struct { cdc codec.Codec db *database.Db messagesParser messages.MessageAddressesParser + source source.Source } // NewModule builds a new Module instance -func NewModule(messagesParser messages.MessageAddressesParser, cdc codec.Codec, db *database.Db) *Module { +func NewModule(source source.Source, messagesParser messages.MessageAddressesParser, cdc codec.Codec, db *database.Db) *Module { return &Module{ messagesParser: messagesParser, + source: source, cdc: cdc, db: db, } diff --git a/modules/auth/source/local/source.go b/modules/auth/source/local/source.go new file mode 100644 index 000000000..5155e0ad4 --- /dev/null +++ b/modules/auth/source/local/source.go @@ -0,0 +1,33 @@ +package local + +import ( + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + + "github.com/forbole/juno/v3/node/local" + + source "github.com/forbole/bdjuno/v3/modules/auth/source" +) + +var ( + _ source.Source = &Source{} +) + +// Source represents the implementation of the bank keeper that works on a local node +type Source struct { + *local.Source + q authtypes.QueryServer +} + +// NewSource builds a new Source instance +func NewSource(source *local.Source, bk authtypes.QueryServer) *Source { + return &Source{ + Source: source, + q: bk, + } +} + +func (s Source) GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) { + + return nil, nil +} diff --git a/modules/auth/source/remote/source.go b/modules/auth/source/remote/source.go new file mode 100644 index 000000000..11aefde92 --- /dev/null +++ b/modules/auth/source/remote/source.go @@ -0,0 +1,69 @@ +package remote + +import ( + "fmt" + + "github.com/cosmos/cosmos-sdk/types/query" + "github.com/forbole/juno/v3/node/remote" + "github.com/rs/zerolog/log" + + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + source "github.com/forbole/bdjuno/v3/modules/auth/source" +) + +var ( + _ source.Source = &Source{} +) + +type Source struct { + *remote.Source + authClient authtypes.QueryClient +} + +// NewSource builds a new Source instance +func NewSource(source *remote.Source, authClient authtypes.QueryClient) *Source { + return &Source{ + Source: source, + authClient: authClient, + } +} + +func (s Source) GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) { + log.Debug().Msg("getting all accounts") + ctx := remote.GetHeightRequestContext(s.Ctx, height) + + var accounts []*codectypes.Any + var nextKey []byte + var stop = false + var counter uint64 + var totalCounts uint64 + var pageLimit uint64 = 1000 + + for !stop { + res, err := s.authClient.Accounts( + ctx, + &authtypes.QueryAccountsRequest{ + Pagination: &query.PageRequest{ + Key: nextKey, + Limit: pageLimit, // Query 100 supplies at time + CountTotal: true, + }, + }) + if err != nil { + return nil, fmt.Errorf("error while getting any accounts from source: %s", err) + } + counter += uint64(len(res.Accounts)) + if res.Pagination.GetTotal() != 0 { + totalCounts = res.Pagination.GetTotal() + } + log.Debug().Uint64("total any account", totalCounts).Uint64("current counter", counter).Msg("getting accounts...") + + nextKey = res.Pagination.NextKey + stop = len(res.Pagination.NextKey) == 0 + + accounts = append(accounts, res.Accounts...) + } + + return accounts, nil +} diff --git a/modules/auth/source/source.go b/modules/auth/source/source.go new file mode 100644 index 000000000..490015031 --- /dev/null +++ b/modules/auth/source/source.go @@ -0,0 +1,7 @@ +package source + +import codectypes "github.com/cosmos/cosmos-sdk/codec/types" + +type Source interface { + GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) +} diff --git a/modules/auth/utils_refresh_all_base_accounts.go b/modules/auth/utils_refresh_all_base_accounts.go new file mode 100644 index 000000000..e068a3e02 --- /dev/null +++ b/modules/auth/utils_refresh_all_base_accounts.go @@ -0,0 +1,42 @@ +package auth + +import ( + "fmt" + + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + "github.com/forbole/bdjuno/v3/types" +) + +func (m *Module) GetAllBaseAccounts(height int64) ([]types.Account, error) { + anyAccounts, err := m.source.GetAllAnyAccounts(height) + if err != nil { + return nil, fmt.Errorf("error while getting any accounts: %s", err) + } + unpacked, err := m.unpackAnyAccounts(anyAccounts) + if err != nil { + return nil, err + } + + return unpacked, nil + +} + +func (m *Module) unpackAnyAccounts(anyAccounts []*codectypes.Any) ([]types.Account, error) { + accounts := []types.Account{} + for _, account := range anyAccounts { + var accountI authtypes.AccountI + err := m.cdc.UnpackAny(account, &accountI) + if err != nil { + return nil, fmt.Errorf("error while unpacking any account: %s", err) + } + + // Get only base accounts + if baseAccount, ok := accountI.(*authtypes.BaseAccount); ok { + accounts = append(accounts, types.NewAccount(baseAccount.Address)) + } + } + + return accounts, nil + +} diff --git a/modules/registrar.go b/modules/registrar.go index 2ec065819..36d145cb4 100644 --- a/modules/registrar.go +++ b/modules/registrar.go @@ -75,7 +75,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules { } actionsModule := actions.NewModule(ctx.JunoConfig, ctx.EncodingConfig) - authModule := auth.NewModule(r.parser, cdc, db) + authModule := auth.NewModule(sources.AuthSource, r.parser, cdc, db) bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db) consensusModule := consensus.NewModule(db) dailyRefetchModule := dailyrefetch.NewModule(ctx.Proxy, db) diff --git a/modules/top_accounts/handle_msg.go b/modules/top_accounts/handle_msg.go index 9d8b37862..6f8233e8f 100644 --- a/modules/top_accounts/handle_msg.go +++ b/modules/top_accounts/handle_msg.go @@ -30,7 +30,7 @@ func (m *Module) HandleMsg(index int, msg sdk.Msg, tx *juno.Tx) error { return fmt.Errorf("error while updating account available balances: %s", err) } - err = m.refreshTopAccountsSum(addresses) + err = m.RefreshTopAccountsSum(addresses) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while refreshing balance: %s", err) } @@ -62,7 +62,7 @@ func (m *Module) handleMsgDelegate(height int64, delAddr string) error { return fmt.Errorf("error while refreshing delegations while handling MsgDelegate: %s", err) } - err = m.refreshTopAccountsSum([]string{delAddr}) + err = m.RefreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgDelegate: %s", err) } @@ -78,7 +78,7 @@ func (m *Module) handleMsgBeginRedelegate( return fmt.Errorf("error while refreshing redelegations while handling MsgBeginRedelegate: %s", err) } - err = m.refreshTopAccountsSum([]string{delAddr}) + err = m.RefreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgBeginRedelegate: %s", err) } @@ -112,7 +112,7 @@ func (m *Module) handleMsgUndelegate(tx *juno.Tx, index int, delAddr string) err return fmt.Errorf("error while refreshing undelegations while handling MsgUndelegate: %s", err) } - err = m.refreshTopAccountsSum([]string{delAddr}) + err = m.RefreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgUndelegate: %s", err) } @@ -151,7 +151,7 @@ func (m *Module) handleMsgWithdrawDelegatorReward(height int64, delAddr string) return fmt.Errorf("error while updating account available balances with MsgWithdrawDelegatorReward: %s", err) } - err = m.refreshTopAccountsSum([]string{delAddr}) + err = m.RefreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgWithdrawDelegatorReward: %s", err) } diff --git a/modules/top_accounts/handle_periodic_operations.go b/modules/top_accounts/handle_periodic_operations.go index 24c7ab710..facdf4818 100644 --- a/modules/top_accounts/handle_periodic_operations.go +++ b/modules/top_accounts/handle_periodic_operations.go @@ -48,7 +48,7 @@ func (m *Module) RefreshRewards() error { return fmt.Errorf("error while refreshing delegators rewards: %s", err) } - err = m.refreshTopAccountsSum(delegators) + err = m.RefreshTopAccountsSum(delegators) if err != nil { return fmt.Errorf("error while refreshing top accounts sum value: %s", err) } diff --git a/modules/top_accounts/utils_refresh.go b/modules/top_accounts/utils_refresh.go index 1bd3989d1..5af4e6450 100644 --- a/modules/top_accounts/utils_refresh.go +++ b/modules/top_accounts/utils_refresh.go @@ -7,7 +7,7 @@ import ( "github.com/rs/zerolog/log" ) -func (m *Module) refreshTopAccountsSum(addresses []string) error { +func (m *Module) RefreshTopAccountsSum(addresses []string) error { for _, addr := range addresses { sum, err := m.db.GetAccountBalanceSum(addr) if err != nil { @@ -60,7 +60,7 @@ func (m *Module) refreshBalance(height int64, address string) func() { Str("operation", "update balance").Msg("error while updating account available balances") } - err = m.refreshTopAccountsSum([]string{address}) + err = m.RefreshTopAccountsSum([]string{address}) if err != nil { log.Error().Str("module", "top_accounts").Err(err). Str("operation", "update top accounts sum").Msg("error while refreshing top accounts sum while refreshing balance") diff --git a/modules/types/sources.go b/modules/types/sources.go index 76fb8560b..1f797522d 100644 --- a/modules/types/sources.go +++ b/modules/types/sources.go @@ -10,6 +10,7 @@ import ( "github.com/cosmos/cosmos-sdk/simapp/params" "github.com/forbole/juno/v3/node/remote" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" distrtypes "github.com/cosmos/cosmos-sdk/x/distribution/types" govtypes "github.com/cosmos/cosmos-sdk/x/gov/types" @@ -21,6 +22,9 @@ import ( nodeconfig "github.com/forbole/juno/v3/node/config" + authsource "github.com/forbole/bdjuno/v3/modules/auth/source" + localauthsource "github.com/forbole/bdjuno/v3/modules/auth/source/local" + remoteauthsource "github.com/forbole/bdjuno/v3/modules/auth/source/remote" banksource "github.com/forbole/bdjuno/v3/modules/bank/source" localbanksource "github.com/forbole/bdjuno/v3/modules/bank/source/local" remotebanksource "github.com/forbole/bdjuno/v3/modules/bank/source/remote" @@ -42,6 +46,7 @@ import ( ) type Sources struct { + AuthSource authsource.Source BankSource banksource.Source DistrSource distrsource.Source GovSource govsource.Source @@ -74,6 +79,7 @@ func buildLocalSources(cfg *local.Details, encodingConfig *params.EncodingConfig ) sources := &Sources{ + AuthSource: localauthsource.NewSource(source, authtypes.QueryServer(app.AccountKeeper)), BankSource: localbanksource.NewSource(source, banktypes.QueryServer(app.BankKeeper)), DistrSource: localdistrsource.NewSource(source, distrtypes.QueryServer(app.DistrKeeper)), GovSource: localgovsource.NewSource(source, govtypes.QueryServer(app.GovKeeper)), @@ -113,6 +119,7 @@ func buildRemoteSources(cfg *remote.Details) (*Sources, error) { } return &Sources{ + AuthSource: remoteauthsource.NewSource(source, authtypes.NewQueryClient(source.GrpcConn)), BankSource: remotebanksource.NewSource(source, banktypes.NewQueryClient(source.GrpcConn)), DistrSource: remotedistrsource.NewSource(source, distrtypes.NewQueryClient(source.GrpcConn)), GovSource: remotegovsource.NewSource(source, govtypes.NewQueryClient(source.GrpcConn)), From 637d6b030578c385e89b149c1b81f4495b9aefd2 Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Tue, 27 Dec 2022 18:24:18 +0800 Subject: [PATCH 02/15] fix comment --- modules/auth/source/remote/source.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/auth/source/remote/source.go b/modules/auth/source/remote/source.go index 11aefde92..a144d29ed 100644 --- a/modules/auth/source/remote/source.go +++ b/modules/auth/source/remote/source.go @@ -38,31 +38,34 @@ func (s Source) GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) { var stop = false var counter uint64 var totalCounts uint64 + + // Get 1000 accounts per query var pageLimit uint64 = 1000 for !stop { + // Get accounts res, err := s.authClient.Accounts( ctx, &authtypes.QueryAccountsRequest{ Pagination: &query.PageRequest{ Key: nextKey, - Limit: pageLimit, // Query 100 supplies at time + Limit: pageLimit, CountTotal: true, }, }) if err != nil { return nil, fmt.Errorf("error while getting any accounts from source: %s", err) } - counter += uint64(len(res.Accounts)) + nextKey = res.Pagination.NextKey + stop = len(res.Pagination.NextKey) == 0 + accounts = append(accounts, res.Accounts...) + + // Log get accounts progress if res.Pagination.GetTotal() != 0 { totalCounts = res.Pagination.GetTotal() } + counter += uint64(len(res.Accounts)) log.Debug().Uint64("total any account", totalCounts).Uint64("current counter", counter).Msg("getting accounts...") - - nextKey = res.Pagination.NextKey - stop = len(res.Pagination.NextKey) == 0 - - accounts = append(accounts, res.Accounts...) } return accounts, nil From bdbecca0935fd2307e10110cc0b2d1e30b9ce39a Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Tue, 27 Dec 2022 18:28:27 +0800 Subject: [PATCH 03/15] add comment --- cmd/parse/top_accounts/all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/parse/top_accounts/all.go b/cmd/parse/top_accounts/all.go index 1946ea888..9a8565517 100644 --- a/cmd/parse/top_accounts/all.go +++ b/cmd/parse/top_accounts/all.go @@ -66,7 +66,7 @@ func allCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { waitGroup.Add(1) - // Get all base accounts + // Get all base accounts, height set to 0 for querying the latest data on chain accounts, err := authModule.GetAllBaseAccounts(0) if err != nil { return fmt.Errorf("error while getting base accounts: %s", err) From e2fdcce76086e9bbd8df3ee9c1f31733386952d6 Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Thu, 29 Dec 2022 21:42:08 +0800 Subject: [PATCH 04/15] add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ea869a41..4bd0d8eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ #### [cheqd] Top Accounts Module - ([\#510](https://github.com/forbole/bdjuno/pull/510)) Implemented `top_accounts` module to store chain native token's balance for ranking +- ([\#511](https://github.com/forbole/bdjuno/pull/511)) Implemented parse cmd for `top_accounts` module #### CI From e0df5f2ac1c33992c6669906aa657b839753163e Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Thu, 29 Dec 2022 21:42:39 +0800 Subject: [PATCH 05/15] fix typo --- cmd/parse/top_accounts/all.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/parse/top_accounts/all.go b/cmd/parse/top_accounts/all.go index 9a8565517..34b7537fd 100644 --- a/cmd/parse/top_accounts/all.go +++ b/cmd/parse/top_accounts/all.go @@ -52,7 +52,7 @@ func allCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { // Build modules authModule := auth.NewModule(sources.AuthSource, nil, parseCtx.EncodingConfig.Marshaler, db) bankModule := bank.NewModule(nil, sources.BankSource, parseCtx.EncodingConfig.Marshaler, db) - distiModule := distribution.NewModule(sources.DistrSource, parseCtx.EncodingConfig.Marshaler, db) + distriModule := distribution.NewModule(sources.DistrSource, parseCtx.EncodingConfig.Marshaler, db) stakingModule := staking.NewModule(sources.StakingSource, parseCtx.EncodingConfig.Marshaler, db) topaccountsModule := topaccounts.NewModule(nil, nil, nil, nil, parseCtx.EncodingConfig.Marshaler, db) @@ -61,7 +61,7 @@ func allCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { workerCount, _ := cmd.Flags().GetInt64(flagWorker) workers := make([]Worker, workerCount) for i := range workers { - workers[i] = NewWorker(exportQueue, bankModule, distiModule, stakingModule, topaccountsModule) + workers[i] = NewWorker(exportQueue, bankModule, distriModule, stakingModule, topaccountsModule) } waitGroup.Add(1) From 679ac6fbf1a2137a849e0c4a3fbb3d67079f1f63 Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Thu, 29 Dec 2022 23:18:13 +0800 Subject: [PATCH 06/15] refine worker --- cmd/parse/top_accounts/all.go | 4 +- cmd/parse/top_accounts/worker.go | 52 +------------------ modules/top_accounts/handle_msg.go | 10 ++-- .../handle_periodic_operations.go | 2 +- modules/top_accounts/utils_refresh.go | 4 +- modules/top_accounts/utils_refresh_all.go | 37 +++++++++++++ 6 files changed, 49 insertions(+), 60 deletions(-) create mode 100644 modules/top_accounts/utils_refresh_all.go diff --git a/cmd/parse/top_accounts/all.go b/cmd/parse/top_accounts/all.go index 34b7537fd..b18a1b896 100644 --- a/cmd/parse/top_accounts/all.go +++ b/cmd/parse/top_accounts/all.go @@ -54,14 +54,14 @@ func allCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { bankModule := bank.NewModule(nil, sources.BankSource, parseCtx.EncodingConfig.Marshaler, db) distriModule := distribution.NewModule(sources.DistrSource, parseCtx.EncodingConfig.Marshaler, db) stakingModule := staking.NewModule(sources.StakingSource, parseCtx.EncodingConfig.Marshaler, db) - topaccountsModule := topaccounts.NewModule(nil, nil, nil, nil, parseCtx.EncodingConfig.Marshaler, db) + topaccountsModule := topaccounts.NewModule(bankModule, distriModule, stakingModule, nil, parseCtx.EncodingConfig.Marshaler, db) // Get workers exportQueue := NewQueue(5) workerCount, _ := cmd.Flags().GetInt64(flagWorker) workers := make([]Worker, workerCount) for i := range workers { - workers[i] = NewWorker(exportQueue, bankModule, distriModule, stakingModule, topaccountsModule) + workers[i] = NewWorker(exportQueue, topaccountsModule) } waitGroup.Add(1) diff --git a/cmd/parse/top_accounts/worker.go b/cmd/parse/top_accounts/worker.go index eca974078..84bc2bd47 100644 --- a/cmd/parse/top_accounts/worker.go +++ b/cmd/parse/top_accounts/worker.go @@ -1,11 +1,6 @@ package top_accounts import ( - "fmt" - - "github.com/forbole/bdjuno/v3/modules/bank" - "github.com/forbole/bdjuno/v3/modules/distribution" - "github.com/forbole/bdjuno/v3/modules/staking" topaccounts "github.com/forbole/bdjuno/v3/modules/top_accounts" "github.com/rs/zerolog/log" ) @@ -18,29 +13,19 @@ func NewQueue(size int) AddressQueue { type Worker struct { queue AddressQueue - bankModule *bank.Module - distriModule *distribution.Module - stakingModule *staking.Module topaccountsModule *topaccounts.Module } -func NewWorker( - queue AddressQueue, - bankModule *bank.Module, distriModule *distribution.Module, - stakingModule *staking.Module, topaccountsModule *topaccounts.Module, -) Worker { +func NewWorker(queue AddressQueue, topaccountsModule *topaccounts.Module) Worker { return Worker{ queue: queue, - bankModule: bankModule, - distriModule: distriModule, - stakingModule: stakingModule, topaccountsModule: topaccountsModule, } } func (w Worker) start() { for address := range w.queue { - err := w.refreshAll(address) + err := w.topaccountsModule.RefreshAll(address) if err != nil { log.Error().Str("account", address).Msg("re-enqueueing failed address") @@ -48,39 +33,6 @@ func (w Worker) start() { w.queue <- address }(address) } - } -} - -func (w *Worker) refreshAll(address string) error { - err := w.bankModule.UpdateBalances([]string{address}, 0) - if err != nil { - return fmt.Errorf("error while refreshing account balance of account %s", address) - } - - err = w.stakingModule.RefreshDelegations(0, address) - if err != nil { - return fmt.Errorf("error while refreshing delegations of account %s", address) - } - err = w.stakingModule.RefreshRedelegations(0, address) - if err != nil { - return fmt.Errorf("error while refreshing redelegations of account %s", address) } - - err = w.stakingModule.RefreshUnbondings(0, address) - if err != nil { - return fmt.Errorf("error while refreshing unbonding delegations of account %s", address) - } - - err = w.distriModule.RefreshDelegatorRewards(0, []string{address}) - if err != nil { - return fmt.Errorf("error while refreshing rewards of account %s", address) - } - - err = w.topaccountsModule.RefreshTopAccountsSum([]string{address}) - if err != nil { - return fmt.Errorf("error while refreshing top account sum of account %s", address) - } - - return nil } diff --git a/modules/top_accounts/handle_msg.go b/modules/top_accounts/handle_msg.go index 6f8233e8f..9d8b37862 100644 --- a/modules/top_accounts/handle_msg.go +++ b/modules/top_accounts/handle_msg.go @@ -30,7 +30,7 @@ func (m *Module) HandleMsg(index int, msg sdk.Msg, tx *juno.Tx) error { return fmt.Errorf("error while updating account available balances: %s", err) } - err = m.RefreshTopAccountsSum(addresses) + err = m.refreshTopAccountsSum(addresses) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while refreshing balance: %s", err) } @@ -62,7 +62,7 @@ func (m *Module) handleMsgDelegate(height int64, delAddr string) error { return fmt.Errorf("error while refreshing delegations while handling MsgDelegate: %s", err) } - err = m.RefreshTopAccountsSum([]string{delAddr}) + err = m.refreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgDelegate: %s", err) } @@ -78,7 +78,7 @@ func (m *Module) handleMsgBeginRedelegate( return fmt.Errorf("error while refreshing redelegations while handling MsgBeginRedelegate: %s", err) } - err = m.RefreshTopAccountsSum([]string{delAddr}) + err = m.refreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgBeginRedelegate: %s", err) } @@ -112,7 +112,7 @@ func (m *Module) handleMsgUndelegate(tx *juno.Tx, index int, delAddr string) err return fmt.Errorf("error while refreshing undelegations while handling MsgUndelegate: %s", err) } - err = m.RefreshTopAccountsSum([]string{delAddr}) + err = m.refreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgUndelegate: %s", err) } @@ -151,7 +151,7 @@ func (m *Module) handleMsgWithdrawDelegatorReward(height int64, delAddr string) return fmt.Errorf("error while updating account available balances with MsgWithdrawDelegatorReward: %s", err) } - err = m.RefreshTopAccountsSum([]string{delAddr}) + err = m.refreshTopAccountsSum([]string{delAddr}) if err != nil { return fmt.Errorf("error while refreshing top accounts sum while handling MsgWithdrawDelegatorReward: %s", err) } diff --git a/modules/top_accounts/handle_periodic_operations.go b/modules/top_accounts/handle_periodic_operations.go index facdf4818..24c7ab710 100644 --- a/modules/top_accounts/handle_periodic_operations.go +++ b/modules/top_accounts/handle_periodic_operations.go @@ -48,7 +48,7 @@ func (m *Module) RefreshRewards() error { return fmt.Errorf("error while refreshing delegators rewards: %s", err) } - err = m.RefreshTopAccountsSum(delegators) + err = m.refreshTopAccountsSum(delegators) if err != nil { return fmt.Errorf("error while refreshing top accounts sum value: %s", err) } diff --git a/modules/top_accounts/utils_refresh.go b/modules/top_accounts/utils_refresh.go index 5af4e6450..1bd3989d1 100644 --- a/modules/top_accounts/utils_refresh.go +++ b/modules/top_accounts/utils_refresh.go @@ -7,7 +7,7 @@ import ( "github.com/rs/zerolog/log" ) -func (m *Module) RefreshTopAccountsSum(addresses []string) error { +func (m *Module) refreshTopAccountsSum(addresses []string) error { for _, addr := range addresses { sum, err := m.db.GetAccountBalanceSum(addr) if err != nil { @@ -60,7 +60,7 @@ func (m *Module) refreshBalance(height int64, address string) func() { Str("operation", "update balance").Msg("error while updating account available balances") } - err = m.RefreshTopAccountsSum([]string{address}) + err = m.refreshTopAccountsSum([]string{address}) if err != nil { log.Error().Str("module", "top_accounts").Err(err). Str("operation", "update top accounts sum").Msg("error while refreshing top accounts sum while refreshing balance") diff --git a/modules/top_accounts/utils_refresh_all.go b/modules/top_accounts/utils_refresh_all.go new file mode 100644 index 000000000..40670a354 --- /dev/null +++ b/modules/top_accounts/utils_refresh_all.go @@ -0,0 +1,37 @@ +package top_accounts + +import "fmt" + +func (m *Module) RefreshAll(address string) error { + err := m.bankModule.UpdateBalances([]string{address}, 0) + if err != nil { + return fmt.Errorf("error while refreshing account balance of account %s", address) + } + + err = m.stakingModule.RefreshDelegations(0, address) + if err != nil { + return fmt.Errorf("error while refreshing delegations of account %s", address) + } + + err = m.stakingModule.RefreshRedelegations(0, address) + if err != nil { + return fmt.Errorf("error while refreshing redelegations of account %s", address) + } + + err = m.stakingModule.RefreshUnbondings(0, address) + if err != nil { + return fmt.Errorf("error while refreshing unbonding delegations of account %s", address) + } + + err = m.distrModule.RefreshDelegatorRewards(0, []string{address}) + if err != nil { + return fmt.Errorf("error while refreshing rewards of account %s", address) + } + + err = m.refreshTopAccountsSum([]string{address}) + if err != nil { + return fmt.Errorf("error while refreshing top account sum of account %s", address) + } + + return nil +} From 2c06a2da1a738d4df3280c84f2dbd3ac61cf8243 Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Thu, 29 Dec 2022 23:20:54 +0800 Subject: [PATCH 07/15] fix typo --- modules/auth/source/local/source.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/auth/source/local/source.go b/modules/auth/source/local/source.go index 5155e0ad4..43f9d2497 100644 --- a/modules/auth/source/local/source.go +++ b/modules/auth/source/local/source.go @@ -20,10 +20,10 @@ type Source struct { } // NewSource builds a new Source instance -func NewSource(source *local.Source, bk authtypes.QueryServer) *Source { +func NewSource(source *local.Source, q authtypes.QueryServer) *Source { return &Source{ Source: source, - q: bk, + q: q, } } From 52c1df319ccf322de0c36860bc608c8f7a3d36ef Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Thu, 29 Dec 2022 23:22:49 +0800 Subject: [PATCH 08/15] add local source --- modules/auth/source/local/source.go | 47 +++++++++++++++++++++++++++- modules/auth/source/remote/source.go | 2 +- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/modules/auth/source/local/source.go b/modules/auth/source/local/source.go index 43f9d2497..0936cc7e2 100644 --- a/modules/auth/source/local/source.go +++ b/modules/auth/source/local/source.go @@ -1,11 +1,16 @@ package local import ( + "fmt" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + "github.com/cosmos/cosmos-sdk/types/query" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + "github.com/rs/zerolog/log" "github.com/forbole/juno/v3/node/local" + sdk "github.com/cosmos/cosmos-sdk/types" source "github.com/forbole/bdjuno/v3/modules/auth/source" ) @@ -28,6 +33,46 @@ func NewSource(source *local.Source, q authtypes.QueryServer) *Source { } func (s Source) GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) { + log.Debug().Msg("getting all accounts") + ctx, err := s.LoadHeight(height) + if err != nil { + return nil, fmt.Errorf("error while loading height: %s", err) + } + + var accounts []*codectypes.Any + var nextKey []byte + var stop = false + var counter uint64 + var totalCounts uint64 + + // Get 1000 accounts per query + var pageLimit uint64 = 1000 + + for !stop { + // Get accounts + res, err := s.q.Accounts( + sdk.WrapSDKContext(ctx), + &authtypes.QueryAccountsRequest{ + Pagination: &query.PageRequest{ + Key: nextKey, + Limit: pageLimit, + CountTotal: true, + }, + }) + if err != nil { + return nil, fmt.Errorf("error while getting any accounts from source: %s", err) + } + nextKey = res.Pagination.NextKey + stop = len(res.Pagination.NextKey) == 0 + accounts = append(accounts, res.Accounts...) + + // Log getting accounts progress + if res.Pagination.GetTotal() != 0 { + totalCounts = res.Pagination.GetTotal() + } + counter += uint64(len(res.Accounts)) + log.Debug().Uint64("total any account", totalCounts).Uint64("current counter", counter).Msg("getting accounts...") + } - return nil, nil + return accounts, nil } diff --git a/modules/auth/source/remote/source.go b/modules/auth/source/remote/source.go index a144d29ed..000b6b624 100644 --- a/modules/auth/source/remote/source.go +++ b/modules/auth/source/remote/source.go @@ -60,7 +60,7 @@ func (s Source) GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) { stop = len(res.Pagination.NextKey) == 0 accounts = append(accounts, res.Accounts...) - // Log get accounts progress + // Log getting accounts progress if res.Pagination.GetTotal() != 0 { totalCounts = res.Pagination.GetTotal() } From d2a844b897a1df3c14c163a5ee3087abc2338ee1 Mon Sep 17 00:00:00 2001 From: huichiaotsou Date: Fri, 30 Dec 2022 15:56:06 +0800 Subject: [PATCH 09/15] use accountI.GetAddress().String() --- modules/auth/utils_refresh_all_base_accounts.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/auth/utils_refresh_all_base_accounts.go b/modules/auth/utils_refresh_all_base_accounts.go index e068a3e02..2e51ec2bd 100644 --- a/modules/auth/utils_refresh_all_base_accounts.go +++ b/modules/auth/utils_refresh_all_base_accounts.go @@ -31,10 +31,7 @@ func (m *Module) unpackAnyAccounts(anyAccounts []*codectypes.Any) ([]types.Accou return nil, fmt.Errorf("error while unpacking any account: %s", err) } - // Get only base accounts - if baseAccount, ok := accountI.(*authtypes.BaseAccount); ok { - accounts = append(accounts, types.NewAccount(baseAccount.Address)) - } + accounts = append(accounts, types.NewAccount(accountI.GetAddress().String())) } return accounts, nil From 01244f100337a8f0de991844f2a89f814baf3f48 Mon Sep 17 00:00:00 2001 From: Magic Cat <37407870+MonikaCat@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:13:31 +0800 Subject: [PATCH 10/15] Update modules/auth/source/local/source.go --- modules/auth/source/local/source.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/auth/source/local/source.go b/modules/auth/source/local/source.go index 0936cc7e2..3a365cca7 100644 --- a/modules/auth/source/local/source.go +++ b/modules/auth/source/local/source.go @@ -18,7 +18,7 @@ var ( _ source.Source = &Source{} ) -// Source represents the implementation of the bank keeper that works on a local node +// Source implements authsource.Source by using a local node type Source struct { *local.Source q authtypes.QueryServer From c94ed185362c5a567a6e4196f6c74e356435389b Mon Sep 17 00:00:00 2001 From: Magic Cat <37407870+MonikaCat@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:13:42 +0800 Subject: [PATCH 11/15] Update modules/top_accounts/utils_refresh_all.go --- modules/top_accounts/utils_refresh_all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/top_accounts/utils_refresh_all.go b/modules/top_accounts/utils_refresh_all.go index 40670a354..a31798d02 100644 --- a/modules/top_accounts/utils_refresh_all.go +++ b/modules/top_accounts/utils_refresh_all.go @@ -5,7 +5,7 @@ import "fmt" func (m *Module) RefreshAll(address string) error { err := m.bankModule.UpdateBalances([]string{address}, 0) if err != nil { - return fmt.Errorf("error while refreshing account balance of account %s", address) + return fmt.Errorf("error while refreshing balance of account %s", address) } err = m.stakingModule.RefreshDelegations(0, address) From 1fc1a9816a82eee57585ce208a09b0171fd648c7 Mon Sep 17 00:00:00 2001 From: Magic Cat <37407870+MonikaCat@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:13:50 +0800 Subject: [PATCH 12/15] Update modules/top_accounts/utils_refresh_all.go --- modules/top_accounts/utils_refresh_all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/top_accounts/utils_refresh_all.go b/modules/top_accounts/utils_refresh_all.go index a31798d02..56dc99f9a 100644 --- a/modules/top_accounts/utils_refresh_all.go +++ b/modules/top_accounts/utils_refresh_all.go @@ -30,7 +30,7 @@ func (m *Module) RefreshAll(address string) error { err = m.refreshTopAccountsSum([]string{address}) if err != nil { - return fmt.Errorf("error while refreshing top account sum of account %s", address) + return fmt.Errorf("error while refreshing top accounts sum") } return nil From c4196b48ea32271fafc97f6f4d2230b5a349541a Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Fri, 3 Feb 2023 13:14:51 +0700 Subject: [PATCH 13/15] updated worker.go --- cmd/parse/top_accounts/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/parse/top_accounts/worker.go b/cmd/parse/top_accounts/worker.go index 84bc2bd47..3c3a6f661 100644 --- a/cmd/parse/top_accounts/worker.go +++ b/cmd/parse/top_accounts/worker.go @@ -27,7 +27,7 @@ func (w Worker) start() { for address := range w.queue { err := w.topaccountsModule.RefreshAll(address) if err != nil { - log.Error().Str("account", address).Msg("re-enqueueing failed address") + log.Error().Str("account", address).Err(err).Msg("re-enqueueing failed address") go func(address string) { w.queue <- address From c9aa4c1302300c584f849573bf55ba191b9bc359 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Fri, 3 Feb 2023 13:32:25 +0700 Subject: [PATCH 14/15] updated err display --- modules/top_accounts/utils_refresh_all.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/top_accounts/utils_refresh_all.go b/modules/top_accounts/utils_refresh_all.go index 56dc99f9a..9209ab49a 100644 --- a/modules/top_accounts/utils_refresh_all.go +++ b/modules/top_accounts/utils_refresh_all.go @@ -5,32 +5,32 @@ import "fmt" func (m *Module) RefreshAll(address string) error { err := m.bankModule.UpdateBalances([]string{address}, 0) if err != nil { - return fmt.Errorf("error while refreshing balance of account %s", address) + return fmt.Errorf("error while refreshing balance of account %s, error: %s", address, err) } err = m.stakingModule.RefreshDelegations(0, address) if err != nil { - return fmt.Errorf("error while refreshing delegations of account %s", address) + return fmt.Errorf("error while refreshing delegations of account %s, error: %s", address, err) } err = m.stakingModule.RefreshRedelegations(0, address) if err != nil { - return fmt.Errorf("error while refreshing redelegations of account %s", address) + return fmt.Errorf("error while refreshing redelegations of account %s, error: %s", address, err) } err = m.stakingModule.RefreshUnbondings(0, address) if err != nil { - return fmt.Errorf("error while refreshing unbonding delegations of account %s", address) + return fmt.Errorf("error while refreshing unbonding delegations of account %s, error: %s", address, err) } err = m.distrModule.RefreshDelegatorRewards(0, []string{address}) if err != nil { - return fmt.Errorf("error while refreshing rewards of account %s", address) + return fmt.Errorf("error while refreshing rewards of account %s, error: %s", address, err) } err = m.refreshTopAccountsSum([]string{address}) if err != nil { - return fmt.Errorf("error while refreshing top accounts sum") + return fmt.Errorf("error while refreshing top accounts sum %s, error: %s", address, err) } return nil From 9b4d1d3b90361fece953db0eb9e9252ee0eb80a7 Mon Sep 17 00:00:00 2001 From: Magic Cat Date: Fri, 3 Feb 2023 19:09:40 +0700 Subject: [PATCH 15/15] updated log msg --- cmd/parse/top_accounts/all.go | 1 + modules/auth/source/local/source.go | 2 +- modules/auth/source/remote/source.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/parse/top_accounts/all.go b/cmd/parse/top_accounts/all.go index b18a1b896..5441c07d7 100644 --- a/cmd/parse/top_accounts/all.go +++ b/cmd/parse/top_accounts/all.go @@ -72,6 +72,7 @@ func allCmd(parseConfig *parsecmdtypes.Config) *cobra.Command { return fmt.Errorf("error while getting base accounts: %s", err) } + log.Debug().Int("total", len(accounts)).Msg("saving accounts...") // Store accounts err = db.SaveAccounts(accounts) if err != nil { diff --git a/modules/auth/source/local/source.go b/modules/auth/source/local/source.go index 3a365cca7..febd8b555 100644 --- a/modules/auth/source/local/source.go +++ b/modules/auth/source/local/source.go @@ -71,7 +71,7 @@ func (s Source) GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) { totalCounts = res.Pagination.GetTotal() } counter += uint64(len(res.Accounts)) - log.Debug().Uint64("total any account", totalCounts).Uint64("current counter", counter).Msg("getting accounts...") + log.Debug().Uint64("total accounts", totalCounts).Uint64("current counter", counter).Msg("getting accounts...") } return accounts, nil diff --git a/modules/auth/source/remote/source.go b/modules/auth/source/remote/source.go index 000b6b624..8f3848e90 100644 --- a/modules/auth/source/remote/source.go +++ b/modules/auth/source/remote/source.go @@ -65,7 +65,7 @@ func (s Source) GetAllAnyAccounts(height int64) ([]*codectypes.Any, error) { totalCounts = res.Pagination.GetTotal() } counter += uint64(len(res.Accounts)) - log.Debug().Uint64("total any account", totalCounts).Uint64("current counter", counter).Msg("getting accounts...") + log.Debug().Uint64("total accounts", totalCounts).Uint64("current counter", counter).Msg("getting accounts...") } return accounts, nil