From 05a59e9bfb12d38696919b38fcf92463664e20f0 Mon Sep 17 00:00:00 2001 From: peterlimg Date: Sat, 30 Sep 2023 17:14:48 +1000 Subject: [PATCH 1/6] Add missing node stat for miners --- code/go/0chain.net/miner/worker.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/code/go/0chain.net/miner/worker.go b/code/go/0chain.net/miner/worker.go index 7a61c460c0..3b24607a09 100644 --- a/code/go/0chain.net/miner/worker.go +++ b/code/go/0chain.net/miner/worker.go @@ -16,10 +16,16 @@ import ( "0chain.net/smartcontract/minersc" "github.com/0chain/common/core/logging" "github.com/0chain/common/core/util" + "github.com/rcrowley/go-metrics" ) const minerScMinerHealthCheck = "miner_health_check" +var ( + missingNodesCount = metrics.GetOrRegisterCounter("missing_nodes_count", nil) + missingNodesTimer = metrics.GetOrRegisterTimer("time_to_get_missing_nodes", nil) +) + /*SetupWorkers - Setup the miner's workers */ func SetupWorkers(ctx context.Context) { mc := GetMinerChain() @@ -291,12 +297,19 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) { for { logging.Logger.Debug("sync all missing nodes - loading all missing nodes...") var err error + start := time.Now() missingNodes, err = lfb.ClientState.GetAllMissingNodes() + elapsed := time.Since(start) if err != nil { logging.Logger.Error("sync all missing nodes - get all missing nodes failed", zap.Error(err)) time.Sleep(3 * time.Second) continue } + + // Record the number of missing nodes and the time it took to acquire them + missingNodesCount.Inc(int64(len(missingNodes))) + missingNodesTimer.Update(elapsed) + logging.Logger.Debug("sync all missing nodes - finish load all missing nodes", zap.Int("num", len(missingNodes))) From 6135f354f2eb3b2e85810e518c3e8312bf97a2aa Mon Sep 17 00:00:00 2001 From: peterlimg Date: Sat, 30 Sep 2023 21:04:41 +1000 Subject: [PATCH 2/6] Add missing node stat metrics --- code/go/0chain.net/miner/worker.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/code/go/0chain.net/miner/worker.go b/code/go/0chain.net/miner/worker.go index 3b24607a09..296c22ea7b 100644 --- a/code/go/0chain.net/miner/worker.go +++ b/code/go/0chain.net/miner/worker.go @@ -22,8 +22,9 @@ import ( const minerScMinerHealthCheck = "miner_health_check" var ( - missingNodesCount = metrics.GetOrRegisterCounter("missing_nodes_count", nil) - missingNodesTimer = metrics.GetOrRegisterTimer("time_to_get_missing_nodes", nil) + missingNodesCount = metrics.GetOrRegisterCounter("missing_nodes_count", nil) + missingNodesTimer = metrics.GetOrRegisterTimer("time_to_get_missing_nodes", nil) + missingNodesSyncTimer = metrics.GetOrRegisterTimer("time_to_sync_missing_nodes", nil) ) /*SetupWorkers - Setup the miner's workers */ @@ -299,7 +300,6 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) { var err error start := time.Now() missingNodes, err = lfb.ClientState.GetAllMissingNodes() - elapsed := time.Since(start) if err != nil { logging.Logger.Error("sync all missing nodes - get all missing nodes failed", zap.Error(err)) time.Sleep(3 * time.Second) @@ -308,7 +308,7 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) { // Record the number of missing nodes and the time it took to acquire them missingNodesCount.Inc(int64(len(missingNodes))) - missingNodesTimer.Update(elapsed) + missingNodesTimer.UpdateSince(start) logging.Logger.Debug("sync all missing nodes - finish load all missing nodes", zap.Int("num", len(missingNodes))) @@ -330,6 +330,7 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) { var ( batchSize = 100 batchs = len(missingNodes) / batchSize + start = time.Now() ) for idx := 1; idx <= batchs; idx++ { @@ -346,6 +347,8 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) { tk.Reset(2 * time.Second) } + missingNodesSyncTimer.UpdateSince(start) + mod := len(missingNodes) % batchSize if mod > 0 { wc := make(chan struct{}, 1) From 7564e551dca8ed604b4194ec8c934a53fa9d782f Mon Sep 17 00:00:00 2001 From: peterlimg Date: Sun, 1 Oct 2023 10:47:07 +1100 Subject: [PATCH 3/6] Present missing nodes in diagnostics page --- code/go/0chain.net/chaincore/chain/entity.go | 14 +++++++ code/go/0chain.net/chaincore/chain/handler.go | 38 +++++++++++++++++++ code/go/0chain.net/miner/worker.go | 13 ++----- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/code/go/0chain.net/chaincore/chain/entity.go b/code/go/0chain.net/chaincore/chain/entity.go index a55e03ccf5..e31addfaf0 100644 --- a/code/go/0chain.net/chaincore/chain/entity.go +++ b/code/go/0chain.net/chaincore/chain/entity.go @@ -16,6 +16,7 @@ import ( "0chain.net/smartcontract/stakepool" "0chain.net/smartcontract/stakepool/spenum" "github.com/0chain/common/core/currency" + "github.com/rcrowley/go-metrics" cstate "0chain.net/chaincore/chain/state" "0chain.net/smartcontract/faucetsc" @@ -207,12 +208,20 @@ type Chain struct { blockSyncC map[string]chan chan *block.Block bscMutex *sync.Mutex + MissingNodesStat *missingNodeStat `json:"-"` + // compute state computeBlockStateC chan struct{} OnBlockAdded func(b *block.Block) } +type missingNodeStat struct { + Counter metrics.Counter + Timer metrics.Timer + SyncTimer metrics.Timer +} + type syncPathNodes struct { round int64 keys []util.Key @@ -503,6 +512,11 @@ func (c *Chain) Initialize() { c.MagicBlockStorage = round.NewRoundStartingStorage() c.OnBlockAdded = func(b *block.Block) { } + c.MissingNodesStat = &missingNodeStat{ + Counter: metrics.GetOrRegisterCounter("missing_nodes_count", nil), + Timer: metrics.GetOrRegisterTimer("time_to_get_missing_nodes", nil), + SyncTimer: metrics.GetOrRegisterTimer("time_to_sync_missing_nodes", nil), + } } /*SetupEntity - setup the entity */ diff --git a/code/go/0chain.net/chaincore/chain/handler.go b/code/go/0chain.net/chaincore/chain/handler.go index 3ead928470..655fe1cbea 100644 --- a/code/go/0chain.net/chaincore/chain/handler.go +++ b/code/go/0chain.net/chaincore/chain/handler.go @@ -33,6 +33,7 @@ import ( "0chain.net/core/datastore" "0chain.net/core/memorystore" "github.com/0chain/common/core/util" + metrics "github.com/rcrowley/go-metrics" "github.com/0chain/common/core/logging" @@ -1709,9 +1710,46 @@ func (c *Chain) MinerStatsHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%v%v", nd.GetPseudoName(), ms.VerificationFailures) } fmt.Fprintf(w, "") + + fmt.Fprintf(w, "
") + fmt.Fprintf(w, "") + fmt.Fprintf(w, "") + fmt.Fprintf(w, "", c.MissingNodesStat.Counter.Count()) + + fmt.Fprintf(w, "") + WriteTimerStatistics(w, c.MissingNodesStat.Timer, 10000) + + fmt.Fprintf(w, "") + WriteTimerStatistics(w, c.MissingNodesStat.SyncTimer, 10000) + + fmt.Fprintf(w, "
Missing node stat
Total count%d
Time to find missing nodes
Time to sync missing nodes
") } } +func WriteTimerStatistics(w http.ResponseWriter, timer metrics.Timer, scaleBy float64) { + scale := func(n float64) float64 { + return (n / scaleBy) + } + percentiles := []float64{0.5, 0.9, 0.95, 0.99, 0.999} + pvals := timer.Percentiles(percentiles) + fmt.Fprintf(w, "") + fmt.Fprintf(w, "") + fmt.Fprintf(w, "", timer.Count()) + fmt.Fprintf(w, "") + fmt.Fprintf(w, "", scale(float64(timer.Min()))) + fmt.Fprintf(w, "", scale(timer.Mean()), scale(timer.StdDev())) + fmt.Fprintf(w, "", scale(float64(timer.Max()))) + for idx, p := range percentiles { + fmt.Fprintf(w, "", 100*p, scale(pvals[idx])) + } + fmt.Fprintf(w, "") + fmt.Fprintf(w, "", timer.Rate1()) + fmt.Fprintf(w, "", timer.Rate5()) + fmt.Fprintf(w, "", timer.Rate15()) + fmt.Fprintf(w, "", timer.RateMean()) + fmt.Fprintf(w, "
Metrics
Count%v
Time taken
Min%.2f ms
Mean%.2f ±%.2f ms
Max%.2f ms
%.2f%%%.2f ms
Rate per second
Last 1-min rate%.2f
Last 5-min rate%.2f
Last 15-min rate%.2f
Overall mean rate%.2f
") +} + func txnIterHandlerFunc(w http.ResponseWriter, lfb *block.Block) func(context.Context, datastore.CollectionEntity) (bool, error) { return func(ctx context.Context, ce datastore.CollectionEntity) (bool, error) { txn, ok := ce.(*transaction.Transaction) diff --git a/code/go/0chain.net/miner/worker.go b/code/go/0chain.net/miner/worker.go index 296c22ea7b..7926fae80a 100644 --- a/code/go/0chain.net/miner/worker.go +++ b/code/go/0chain.net/miner/worker.go @@ -16,17 +16,10 @@ import ( "0chain.net/smartcontract/minersc" "github.com/0chain/common/core/logging" "github.com/0chain/common/core/util" - "github.com/rcrowley/go-metrics" ) const minerScMinerHealthCheck = "miner_health_check" -var ( - missingNodesCount = metrics.GetOrRegisterCounter("missing_nodes_count", nil) - missingNodesTimer = metrics.GetOrRegisterTimer("time_to_get_missing_nodes", nil) - missingNodesSyncTimer = metrics.GetOrRegisterTimer("time_to_sync_missing_nodes", nil) -) - /*SetupWorkers - Setup the miner's workers */ func SetupWorkers(ctx context.Context) { mc := GetMinerChain() @@ -307,8 +300,8 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) { } // Record the number of missing nodes and the time it took to acquire them - missingNodesCount.Inc(int64(len(missingNodes))) - missingNodesTimer.UpdateSince(start) + mc.MissingNodesStat.Counter.Inc(int64(len(missingNodes))) + mc.MissingNodesStat.Timer.UpdateSince(start) logging.Logger.Debug("sync all missing nodes - finish load all missing nodes", zap.Int("num", len(missingNodes))) @@ -347,7 +340,7 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) { tk.Reset(2 * time.Second) } - missingNodesSyncTimer.UpdateSince(start) + mc.MissingNodesStat.SyncTimer.UpdateSince(start) mod := len(missingNodes) % batchSize if mod > 0 { From f1e5315a0467c7870856cfc62112c7db39a422e6 Mon Sep 17 00:00:00 2001 From: peterlimg Date: Sun, 1 Oct 2023 11:25:50 +1100 Subject: [PATCH 4/6] Upate stat page --- code/go/0chain.net/chaincore/chain/handler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/chaincore/chain/handler.go b/code/go/0chain.net/chaincore/chain/handler.go index 655fe1cbea..e39a51b01b 100644 --- a/code/go/0chain.net/chaincore/chain/handler.go +++ b/code/go/0chain.net/chaincore/chain/handler.go @@ -1712,8 +1712,9 @@ func (c *Chain) MinerStatsHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "") fmt.Fprintf(w, "
") - fmt.Fprintf(w, "") - fmt.Fprintf(w, "") + + fmt.Fprintf(w, "
Missing node stat
") + fmt.Fprintf(w, "
Missing node stat
") fmt.Fprintf(w, "", c.MissingNodesStat.Counter.Count()) fmt.Fprintf(w, "") @@ -1723,6 +1724,7 @@ func (c *Chain) MinerStatsHandler(w http.ResponseWriter, r *http.Request) { WriteTimerStatistics(w, c.MissingNodesStat.SyncTimer, 10000) fmt.Fprintf(w, "
Total count%d
Time to find missing nodes
") + fmt.Fprintf(w, "
 
") } } From b1ea948b87349c1891f1fa7ed8a2e29d2bff7f07 Mon Sep 17 00:00:00 2001 From: peterlimg Date: Sun, 1 Oct 2023 11:39:18 +1100 Subject: [PATCH 5/6] Algin tables --- code/go/0chain.net/chaincore/chain/handler.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/chaincore/chain/handler.go b/code/go/0chain.net/chaincore/chain/handler.go index e39a51b01b..f11e16a5d6 100644 --- a/code/go/0chain.net/chaincore/chain/handler.go +++ b/code/go/0chain.net/chaincore/chain/handler.go @@ -1713,16 +1713,25 @@ func (c *Chain) MinerStatsHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "
") - fmt.Fprintf(w, "
Missing node stat
") + fmt.Fprintf(w, "
Missing Node Stat
") + fmt.Fprintf(w, "") + fmt.Fprintf(w, "") fmt.Fprintf(w, "") + fmt.Fprintf(w, "") fmt.Fprintf(w, "") + fmt.Fprintf(w, "") + fmt.Fprintf(w, "
") fmt.Fprintf(w, "") fmt.Fprintf(w, "", c.MissingNodesStat.Counter.Count()) + fmt.Fprintf(w, "
Total count%d
") + fmt.Fprintf(w, "
Time to find missing nodes
") WriteTimerStatistics(w, c.MissingNodesStat.Timer, 10000) + fmt.Fprintf(w, "
Time to sync missing nodes
") WriteTimerStatistics(w, c.MissingNodesStat.SyncTimer, 10000) + fmt.Fprintf(w, "
") fmt.Fprintf(w, "") fmt.Fprintf(w, "
 
") } From 9527f3aab324b5312ccd7b1b488c42a171f9c3ce Mon Sep 17 00:00:00 2001 From: peterlimg Date: Sun, 1 Oct 2023 11:47:23 +1100 Subject: [PATCH 6/6] Update missing node stat table width --- code/go/0chain.net/chaincore/chain/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/chaincore/chain/handler.go b/code/go/0chain.net/chaincore/chain/handler.go index f11e16a5d6..e04b0c29d4 100644 --- a/code/go/0chain.net/chaincore/chain/handler.go +++ b/code/go/0chain.net/chaincore/chain/handler.go @@ -1714,7 +1714,7 @@ func (c *Chain) MinerStatsHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "
") fmt.Fprintf(w, "
Missing Node Stat
") - fmt.Fprintf(w, "") + fmt.Fprintf(w, "
") fmt.Fprintf(w, "
") fmt.Fprintf(w, "") fmt.Fprintf(w, "", c.MissingNodesStat.Counter.Count())
Total count%d