From e9937e242ea0118de63f6b73691db381e1abada6 Mon Sep 17 00:00:00 2001 From: randyahx <62416962+randyahx@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:22:35 +0800 Subject: [PATCH 1/2] fix: handle events where components cannot be fetched successfully for hash verification (#87) * add logs * Update hash_verifier.go * use lru cache for verifier * fix config (cherry picked from commit 472ad5940966881015a2db83e793ea025e8192bb) * remove logs * update dependency * Update go.mod * update dependencies --- app/app.go | 2 +- config/config.json | 6 ++-- executor/client.go | 4 +-- executor/executor.go | 22 +++++++-------- go.mod | 6 ++-- go.sum | 12 ++++---- monitor/monitor.go | 4 --- verifier/hash_verifier.go | 58 ++++++++++++++++++++++++++++++--------- 8 files changed, 72 insertions(+), 42 deletions(-) diff --git a/app/app.go b/app/app.go index 599f48a..95662c7 100644 --- a/app/app.go +++ b/app/app.go @@ -45,7 +45,7 @@ func NewApp(cfg *config.Config) *App { db, err := gorm.Open(mysql.Open(dbPath), &gorm.Config{}) // only for debug purpose - db = db.Debug() + //db = db.Debug() if err != nil { panic(fmt.Sprintf("open db error, err=%+v", err.Error())) diff --git a/config/config.json b/config/config.json index a2c0543..9343702 100644 --- a/config/config.json +++ b/config/config.json @@ -12,8 +12,10 @@ "chain_id_string": "greenfield_9000-121", "gas_limit": 1000, "fee_amount": "5000000000000", - "fee_denom": "BNB", - "deduplication_interval": 100 + "fee_denom": "BNB" + }, + "metrics_config": { + "port": 8080 }, "log_config": { "level": "DEBUG", diff --git a/executor/client.go b/executor/client.go index 25b731e..4d1bb73 100644 --- a/executor/client.go +++ b/executor/client.go @@ -13,7 +13,7 @@ import ( type JsonRpcClient = *jsonrpcclient.Client type GnfdCompositeClient struct { - gnfdclient.Client + gnfdclient.IClient client.TendermintClient JsonRpcClient Height int64 @@ -36,7 +36,7 @@ func NewGnfdCompositClients(rpcAddrs []string, chainId string, account *types.Ac panic(err) } clients = append(clients, &GnfdCompositeClient{ - Client: sdkClient, + IClient: sdkClient, TendermintClient: client.NewTendermintClient(rpcAddrs[i]), JsonRpcClient: jsonRpcClient, }) diff --git a/executor/executor.go b/executor/executor.go index 355c9ea..8b68f7b 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -120,7 +120,7 @@ func getGreenfieldBlsPrivateKey(cfg *config.GreenfieldConfig) string { func (e *Executor) GetBlockAndBlockResultAtHeight(height int64) (*tmtypes.Block, *ctypes.ResultBlockResults, error) { block, err := e.clients.GetClient().TmClient.Block(context.Background(), &height) if err != nil { - logging.Logger.Errorf("executor failed to get block at height %d, err=%+v", height, err.Error()) + //logging.Logger.Errorf("executor failed to get block at height %d, err=%+v", height, err.Error()) return nil, nil, err } blockResults, err := e.clients.GetClient().TmClient.BlockResults(context.Background(), &height) @@ -132,7 +132,7 @@ func (e *Executor) GetBlockAndBlockResultAtHeight(height int64) (*tmtypes.Block, } func (e *Executor) GetLatestBlockHeight() (uint64, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() res, err := client.GetLatestBlockHeight(context.Background()) latestHeight := uint64(res) if err != nil { @@ -212,7 +212,7 @@ func (e *Executor) GetValidatorsBlsPublicKey() ([]string, error) { } func (e *Executor) QueryInturnAttestationSubmitter() (*challengetypes.QueryInturnAttestationSubmitterResponse, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() res, err := client.InturnAttestationSubmitter(context.Background(), &challengetypes.QueryInturnAttestationSubmitterRequest{}) if err != nil { logging.Logger.Errorf("executor failed to get inturn attestation submitter, err=%+v", err.Error()) @@ -222,7 +222,7 @@ func (e *Executor) QueryInturnAttestationSubmitter() (*challengetypes.QueryIntur } func (e *Executor) AttestChallenge(submitterAddress, challengerAddress, spOperatorAddress string, challengeId uint64, objectId sdkmath.Uint, voteResult challengetypes.VoteResult, voteValidatorSet []uint64, VoteAggSignature []byte, txOption sdktypes.TxOption) (bool, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() logging.Logger.Infof("attest challenge params: submitterAddress=%s, challengerAddress=%s, spOperatorAddress=%s, challengeId=%d, objectId=%s, voteResult=%s, voteValidatorSet=%+v, VoteAggSignature=%+v, txOption=%+v", submitterAddress, challengerAddress, spOperatorAddress, challengeId, objectId.String(), voteResult.String(), voteValidatorSet, VoteAggSignature, txOption) res, err := client.AttestChallenge(context.Background(), submitterAddress, challengerAddress, spOperatorAddress, challengeId, objectId, voteResult, voteValidatorSet, VoteAggSignature, txOption) if err != nil { @@ -242,7 +242,7 @@ func (e *Executor) AttestChallenge(submitterAddress, challengerAddress, spOperat } func (e *Executor) QueryLatestAttestedChallengeIds() ([]uint64, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() res, err := client.LatestAttestedChallenges(context.Background(), &challengetypes.QueryLatestAttestedChallengesRequest{}) if err != nil { @@ -259,7 +259,7 @@ func (e *Executor) QueryLatestAttestedChallengeIds() ([]uint64, error) { } func (e *Executor) queryChallengeHeartbeatInterval() (uint64, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() q := challengetypes.QueryParamsRequest{} res, err := client.ChallengeParams(context.Background(), &q) if err != nil { @@ -288,7 +288,7 @@ func (e *Executor) QueryChallengeHeartbeatInterval() (uint64, error) { } func (e *Executor) QueryChallengeSlashCoolingOffPeriod() (uint64, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() params, err := client.ChallengeParams(context.Background(), &challengetypes.QueryParamsRequest{}) if err != nil { logging.Logger.Errorf("query challenge params failed, err=%+v", err.Error()) @@ -324,7 +324,7 @@ func (e *Executor) GetHeightLoop() { } func (e *Executor) GetStorageProviderEndpoint(address string) (string, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() spAddr, err := sdk.AccAddressFromHexUnsafe(address) if err != nil { logging.Logger.Errorf("error converting addr from hex unsafe when getting sp endpoint, err=%+v", err.Error()) @@ -341,7 +341,7 @@ func (e *Executor) GetStorageProviderEndpoint(address string) (string, error) { } func (e *Executor) GetObjectInfoChecksums(objectId string) ([][]byte, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() res, err := client.HeadObjectByID(context.Background(), objectId) if err != nil { @@ -352,7 +352,7 @@ func (e *Executor) GetObjectInfoChecksums(objectId string) ([][]byte, error) { } func (e *Executor) GetChallengeResultFromSp(objectId, endpoint string, segmentIndex, redundancyIndex int) (*types.ChallengeResult, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() challengeInfoOpts := types.GetChallengeInfoOptions{ Endpoint: endpoint, @@ -398,7 +398,7 @@ func (e *Executor) GetAddr() string { } func (e *Executor) GetNonce() (uint64, error) { - client := e.clients.GetClient().Client + client := e.clients.GetClient() account, err := client.GetAccount(context.Background(), e.GetAddr()) if err != nil { logging.Logger.Errorf("error getting account, err=%+v", err.Error()) diff --git a/go.mod b/go.mod index 5ac7072..f04ca7f 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( cosmossdk.io/math v1.0.1 github.com/avast/retry-go/v4 v4.3.1 github.com/aws/aws-sdk-go v1.40.45 - github.com/bnb-chain/greenfield v0.2.5 + github.com/bnb-chain/greenfield v0.2.6 github.com/bnb-chain/greenfield-common/go v0.0.0-20230830120314-a54ffd6da39f - github.com/bnb-chain/greenfield-go-sdk v0.2.5 + github.com/bnb-chain/greenfield-go-sdk v0.2.6 github.com/cometbft/cometbft v0.37.2 github.com/cosmos/cosmos-sdk v0.47.3 github.com/ethereum/go-ethereum v1.10.26 @@ -173,7 +173,7 @@ replace ( github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v0.0.3 github.com/cometbft/cometbft-db => github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 - github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v0.2.5 + github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v0.2.6 github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1 github.com/ferranbt/fastssz => github.com/prysmaticlabs/fastssz v0.0.0-20220110145812-fafb696cae88 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 diff --git a/go.sum b/go.sum index 3d16e94..e24cb88 100644 --- a/go.sum +++ b/go.sum @@ -166,22 +166,22 @@ github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 h1:41iFGWnSlI2 github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/bnb-chain/greenfield v0.2.5 h1:AmHvGfJmHm7FXB4II4mAB6i68bw98p86ZlIs9r6YeRM= -github.com/bnb-chain/greenfield v0.2.5/go.mod h1:nNsy8QGR8+R0j6Qz/duNE94NDDNuomC6wWDCzfl7jKc= +github.com/bnb-chain/greenfield v0.2.6 h1:U40wmSBQR4Wd0HiVCu/J6zqoLS4YUrvNgyuX71kgK3U= +github.com/bnb-chain/greenfield v0.2.6/go.mod h1:8kGVzKu3BEbpotk2Lmp/OzPh+nhbMsuUJueZtn0he4s= github.com/bnb-chain/greenfield-cometbft v0.0.3 h1:tv8NMy3bzX/1urqXGQIIAZSLy83loiI+dG0VKeyh1CY= github.com/bnb-chain/greenfield-cometbft v0.0.3/go.mod h1:f35mk/r5ab6yvzlqEWZt68LfUje68sYgMpVlt2CUYMk= github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1 h1:XcWulGacHVRiSCx90Q8Y//ajOrLNBQWR/KDB89dy3cU= github.com/bnb-chain/greenfield-cometbft-db v0.8.1-alpha.1/go.mod h1:ey1CiK4bYo1RBNJLRiVbYr5CMdSxci9S/AZRINLtppI= github.com/bnb-chain/greenfield-common/go v0.0.0-20230830120314-a54ffd6da39f h1:zJvB2wCd80DQ9Nh/ZNQiP8MrHygSpDoav7OzHyIi/pM= github.com/bnb-chain/greenfield-common/go v0.0.0-20230830120314-a54ffd6da39f/go.mod h1:it3JJVHeG9Wp4QED2GkY/7V9Qo3BuPdoC5/4/U6ecJM= -github.com/bnb-chain/greenfield-cosmos-sdk v0.2.5 h1:8zIn/ExfMHZVBbVwhILG9KYO1m3pZO8I8stpqbFgzkk= -github.com/bnb-chain/greenfield-cosmos-sdk v0.2.5/go.mod h1:y3hDhQhil5hMIhwBTpu07RZBF30ZITkoE+GHhVZChtY= +github.com/bnb-chain/greenfield-cosmos-sdk v0.2.6 h1:aPrd2nG2nYZr8nSmGgN8efus6sni4OIZ1Bdx6s+mS3A= +github.com/bnb-chain/greenfield-cosmos-sdk v0.2.6/go.mod h1:y3hDhQhil5hMIhwBTpu07RZBF30ZITkoE+GHhVZChtY= github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210 h1:GHPbV2bC+gmuO6/sG0Tm8oGal3KKSRlyE+zPscDjlA8= github.com/bnb-chain/greenfield-cosmos-sdk/api v0.0.0-20230816082903-b48770f5e210/go.mod h1:vhsZxXE9tYJeYB5JR4hPhd6Pc/uPf7j1T8IJ7p9FdeM= github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230816082903-b48770f5e210 h1:FLVOn4+OVbsKi2+YJX5kmD27/4dRu4FW7xCXFhzDO5s= github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230816082903-b48770f5e210/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM= -github.com/bnb-chain/greenfield-go-sdk v0.2.5 h1:ErI7nUtNCRDvpXSVaG/XKXEBhTcmiofg/2ciKU1uBag= -github.com/bnb-chain/greenfield-go-sdk v0.2.5/go.mod h1:rWI9Xgcto7ujajNfG6x7+lW1SzbJWBkIgnOcoNjWS7U= +github.com/bnb-chain/greenfield-go-sdk v0.2.6 h1:CGmMK+Ie3ZXae86rGYJnaF7FmO+y92S4a/Qp7DzxhLU= +github.com/bnb-chain/greenfield-go-sdk v0.2.6/go.mod h1:MrxZcdoK/YC8o0wECOv8fskozBokekxApLbi1P6qeTI= github.com/bnb-chain/greenfield-iavl v0.20.1 h1:y3L64GU99otNp27/xLVBTDbv4eroR6CzoYz0rbaVotM= github.com/bnb-chain/greenfield-iavl v0.20.1/go.mod h1:oLksTs8dfh7DYIKBro7hbRQ+ewls7ghJ27pIXlbEXyI= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= diff --git a/monitor/monitor.go b/monitor/monitor.go index 002f111..e6e9231 100644 --- a/monitor/monitor.go +++ b/monitor/monitor.go @@ -172,10 +172,8 @@ func (m *Monitor) monitorChallengeEvents(block *tmtypes.Block, blockResults *cty func (m *Monitor) calNextHeight() (uint64, error) { latestPolledBlock, err := m.dataProvider.GetLatestBlock() if err != nil && err != gorm.ErrRecordNotFound { - logging.Logger.Errorf("failed to get latest block from db, error: %s", err.Error()) latestHeight, err := m.executor.GetLatestBlockHeight() if err != nil { - logging.Logger.Errorf("failed to get latest block height, error: %s", err.Error()) return 0, err } return latestHeight, err @@ -183,7 +181,6 @@ func (m *Monitor) calNextHeight() (uint64, error) { if latestPolledBlock.Height == 0 { // a fresh database latestHeight, err := m.executor.GetLatestBlockHeight() if err != nil { - logging.Logger.Errorf("failed to get latest block height, error: %s", err.Error()) return m.executor.GetCachedBlockHeight(), err } return latestHeight, nil @@ -192,7 +189,6 @@ func (m *Monitor) calNextHeight() (uint64, error) { latestBlockHeight, err := m.executor.GetLatestBlockHeight() if err != nil { - logging.Logger.Errorf("failed to get latest block height, error: %s", err.Error()) return 0, err } // pauses challenger for a bit since it already caught the newest block diff --git a/verifier/hash_verifier.go b/verifier/hash_verifier.go index 740e01c..883034c 100644 --- a/verifier/hash_verifier.go +++ b/verifier/hash_verifier.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + lru "github.com/hashicorp/golang-lru" "golang.org/x/sync/semaphore" "io" "strings" @@ -25,7 +26,7 @@ type Verifier struct { config *config.Config executor *executor.Executor deduplicationInterval uint64 - cachedChallengeIds map[uint64]bool + cachedChallengeIds *lru.Cache mtx sync.RWMutex dataProvider DataProvider limiterSemaphore *semaphore.Weighted @@ -37,6 +38,9 @@ func NewHashVerifier(cfg *config.Config, executor *executor.Executor, dataProvid ) *Verifier { limiterSemaphore := semaphore.NewWeighted(20) + cacheSize := 1000 + lruCache, _ := lru.New(cacheSize) + deduplicationInterval, err := executor.QueryChallengeSlashCoolingOffPeriod() if err != nil { logging.Logger.Errorf("verifier failed to query slash cooling off period, err=%+v", err) @@ -46,6 +50,7 @@ func NewHashVerifier(cfg *config.Config, executor *executor.Executor, dataProvid config: cfg, executor: executor, deduplicationInterval: deduplicationInterval, + cachedChallengeIds: lruCache, mtx: sync.RWMutex{}, dataProvider: dataProvider, limiterSemaphore: limiterSemaphore, @@ -54,8 +59,6 @@ func NewHashVerifier(cfg *config.Config, executor *executor.Executor, dataProvid } func (v *Verifier) VerifyHashLoop() { - // Event lasts for 300 blocks, 2x for redundancy - v.cachedChallengeIds = make(map[uint64]bool, common.CacheSize) for { err := v.verifyHash() if err != nil { @@ -88,7 +91,7 @@ func (v *Verifier) verifyHash() error { for _, event := range events { v.mtx.Lock() - isCached := v.cachedChallengeIds[event.ChallengeId] + isCached := v.cachedChallengeIds.Contains(event.ChallengeId) v.mtx.Unlock() if isCached { @@ -112,7 +115,7 @@ func (v *Verifier) verifyHash() error { if err != nil { if err.Error() == common.ErrEventExpired.Error() { v.mtx.Lock() - delete(v.cachedChallengeIds, event.ChallengeId) + v.cachedChallengeIds.Remove(event.ChallengeId) v.mtx.Unlock() continue } @@ -121,7 +124,7 @@ func (v *Verifier) verifyHash() error { if !isCached { v.mtx.Lock() - v.cachedChallengeIds[event.ChallengeId] = true + v.cachedChallengeIds.Add(event.ChallengeId, true) v.mtx.Unlock() } } @@ -132,25 +135,54 @@ func (v *Verifier) verifyHash() error { } func (v *Verifier) verifyForSingleEvent(event *model.Event) error { + var err error startTime := time.Now() logging.Logger.Infof("verifier started for challengeId: %d %s", event.ChallengeId, time.Now().Format("15:04:05.000000")) currentHeight := v.executor.GetCachedBlockHeight() - if err := v.preCheck(event, currentHeight); err != nil { + if err = v.preCheck(event, currentHeight); err != nil { return err } - endpoint, err := v.executor.GetStorageProviderEndpoint(event.SpOperatorAddress) + // Retry GetStorageProviderEndpoint and GetObjectInfoChecksums up to 5 times + var endpoint string + _ = retry.Do( + func() error { + endpoint, err = v.executor.GetStorageProviderEndpoint(event.SpOperatorAddress) + if err != nil { + logging.Logger.Errorf("verifier failed to get sp endpoint for challengeId: %s, objectId: %s, err=%+v", event.ChallengeId, event.ObjectId, err.Error()) + } + return err + }, retry.Context(context.Background()), common.RtyAttem, common.RtyDelay, common.RtyErr) + if err != nil { - logging.Logger.Errorf("verifier failed to get sp endpoint for challengeId: %s, objectId: %s, err=%+v", err.Error(), event.ChallengeId, event.ObjectId) + err = v.dataProvider.UpdateEventStatusVerifyResult(event.ChallengeId, model.Verified, model.Unknown) + v.metricService.IncVerifiedChallenges() + v.metricService.IncChallengeFailed() + if err != nil { + return err + } return err } - logging.Logger.Infof("challengeId: %d, sp endpoint: %s, objectId: %s, segmentIndex: %d, redundancyIndex: %d", event.ChallengeId, endpoint, event.ObjectId, event.SegmentIndex, event.RedundancyIndex) // Call blockchain for object info to get original hash - checksums, err := v.executor.GetObjectInfoChecksums(event.ObjectId) + var checksums [][]byte + _ = retry.Do( + func() error { + checksums, err = v.executor.GetObjectInfoChecksums(event.ObjectId) + if err != nil { + if strings.Contains(err.Error(), "No such object") { + logging.Logger.Errorf("No such object error for challengeId: %d", event.ChallengeId) + } + logging.Logger.Errorf("hash verifier error getting object checksums for challengeId: %d, err=%s", event.ChallengeId, err.Error()) + } + return err + }, retry.Context(context.Background()), common.RtyAttem, common.RtyDelay, common.RtyErr) if err != nil { - if strings.Contains(err.Error(), "No such object") { - logging.Logger.Errorf("No such object error for challengeId: %d", event.ChallengeId) + err = v.dataProvider.UpdateEventStatusVerifyResult(event.ChallengeId, model.Verified, model.Unknown) + v.metricService.IncVerifiedChallenges() + v.metricService.IncChallengeFailed() + if err != nil { + return err } return err } From 6eb11bcd5908206f3cceaf2769ac0eb39b67da4a Mon Sep 17 00:00:00 2001 From: randyahx <62416962+randyahx@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:29:44 +0800 Subject: [PATCH 2/2] release: draft release for v0.0.7 (#88) * Update CHANGELOG.md * Update CHANGELOG.md --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad1a80..5e8ec46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## v0.0.7 +This release updates the dependencies to adapt to Greenfield v0.2.6 and adds a bugfix to handle events where components cannot be fetched successfully for hash verification. + +Bugfixes: +* [#87](https://github.com/bnb-chain/greenfield-challenger/pull/87) fix: handle events where components cannot be fetched successfully for hash verification + +## v0.0.6-hf.1 +This is a hotfix release to update the outdated config file. + +Chore: +* [#86](https://github.com/bnb-chain/greenfield-challenger/pull/86) fix: update outdated config file + ## v0.0.6 This release updates the dependencies to adapt to Greenfield v0.2.5 and contains 1 new feature.