Add missing nodes stat in miner

code/go/0chain.net/chaincore/chain/entity.go

@@ -16,6 +16,7 @@ import (
 	"0chain.net/smartcontract/stakepool"
 	"0chain.net/smartcontract/stakepool/spenum"
 	"github.com/0chain/common/core/currency"
+	"github.com/rcrowley/go-metrics"
 
 	cstate "0chain.net/chaincore/chain/state"
 	"0chain.net/smartcontract/faucetsc"
@@ -207,12 +208,20 @@ type Chain struct {
 	blockSyncC            map[string]chan chan *block.Block
 	bscMutex              *sync.Mutex
 
+	MissingNodesStat *missingNodeStat `json:"-"`
+
 	// compute state
 	computeBlockStateC chan struct{}
 
 	OnBlockAdded func(b *block.Block)
 }
 
+type missingNodeStat struct {
+	Counter   metrics.Counter
+	Timer     metrics.Timer
+	SyncTimer metrics.Timer
+}
+
 type syncPathNodes struct {
 	round  int64
 	keys   []util.Key
@@ -503,6 +512,11 @@ func (c *Chain) Initialize() {
 	c.MagicBlockStorage = round.NewRoundStartingStorage()
 	c.OnBlockAdded = func(b *block.Block) {
 	}
+	c.MissingNodesStat = &missingNodeStat{
+		Counter:   metrics.GetOrRegisterCounter("missing_nodes_count", nil),
+		Timer:     metrics.GetOrRegisterTimer("time_to_get_missing_nodes", nil),
+		SyncTimer: metrics.GetOrRegisterTimer("time_to_sync_missing_nodes", nil),
+	}
 }
 
 /*SetupEntity - setup the entity */

code/go/0chain.net/chaincore/chain/handler.go

@@ -33,6 +33,7 @@ import (
 	"0chain.net/core/datastore"
 	"0chain.net/core/memorystore"
 	"github.com/0chain/common/core/util"
+	metrics "github.com/rcrowley/go-metrics"
 
 	"github.com/0chain/common/core/logging"
 
@@ -1709,9 +1710,57 @@ func (c *Chain) MinerStatsHandler(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintf(w, "<tr><td>%v</td><td class='number'>%v</td></tr>", nd.GetPseudoName(), ms.VerificationFailures)
 		}
 		fmt.Fprintf(w, "</table>")
+
+		fmt.Fprintf(w, "<br>")
+
+		fmt.Fprintf(w, "<div>Missing Node Stat</div>")
+		fmt.Fprintf(w, "<table style='width:500'>")
+		fmt.Fprintf(w, "<tr><td colspan='3' style='text-align:center'>")
+		fmt.Fprintf(w, "<table style='width:100%%;'>")
+		fmt.Fprintf(w, "<tr><td>Total count</td><td>%d</td></tr>", c.MissingNodesStat.Counter.Count())
+		fmt.Fprintf(w, "</table>")
+		fmt.Fprintf(w, "</td></tr>")
+
+		fmt.Fprintf(w, "<tr><td>Time to find missing nodes</td></tr>")
+		fmt.Fprintf(w, "<tr><td colspan='3' style='text-align:center'>")
+		WriteTimerStatistics(w, c.MissingNodesStat.Timer, 10000)
+		fmt.Fprintf(w, "</td></tr>")
+
+		fmt.Fprintf(w, "<tr><td>Time to sync missing nodes</td></tr>")
+		fmt.Fprintf(w, "<tr><td colspan='3' style='text-align:center'>")
+		WriteTimerStatistics(w, c.MissingNodesStat.SyncTimer, 10000)
+		fmt.Fprintf(w, "</td></tr>")
+
+		fmt.Fprintf(w, "</table>")
+		fmt.Fprintf(w, "</table>")
+		fmt.Fprintf(w, "<div>&nbsp;</div>")
 	}
 }
 
+func WriteTimerStatistics(w http.ResponseWriter, timer metrics.Timer, scaleBy float64) {
+	scale := func(n float64) float64 {
+		return (n / scaleBy)
+	}
+	percentiles := []float64{0.5, 0.9, 0.95, 0.99, 0.999}
+	pvals := timer.Percentiles(percentiles)
+	fmt.Fprintf(w, "<table width='100%%'>")
+	fmt.Fprintf(w, "<tr><td class='sheader' colspan=2'>Metrics</td></tr>")
+	fmt.Fprintf(w, "<tr><td>Count</td><td>%v</td></tr>", timer.Count())
+	fmt.Fprintf(w, "<tr><td class='sheader' colspan='2'>Time taken</td></tr>")
+	fmt.Fprintf(w, "<tr><td>Min</td><td>%.2f ms</td></tr>", scale(float64(timer.Min())))
+	fmt.Fprintf(w, "<tr><td>Mean</td><td>%.2f &plusmn;%.2f ms</td></tr>", scale(timer.Mean()), scale(timer.StdDev()))
+	fmt.Fprintf(w, "<tr><td>Max</td><td>%.2f ms</td></tr>", scale(float64(timer.Max())))
+	for idx, p := range percentiles {
+		fmt.Fprintf(w, "<tr><td>%.2f%%</td><td>%.2f ms</td></tr>", 100*p, scale(pvals[idx]))
+	}
+	fmt.Fprintf(w, "<tr><td class='sheader' colspan='2'>Rate per second</td></tr>")
+	fmt.Fprintf(w, "<tr><td>Last 1-min rate</td><td>%.2f</td></tr>", timer.Rate1())
+	fmt.Fprintf(w, "<tr><td>Last 5-min rate</td><td>%.2f</td></tr>", timer.Rate5())
+	fmt.Fprintf(w, "<tr><td>Last 15-min rate</td><td>%.2f</td></tr>", timer.Rate15())
+	fmt.Fprintf(w, "<tr><td>Overall mean rate</td><td>%.2f</td></tr>", timer.RateMean())
+	fmt.Fprintf(w, "</table>")
+}
+
 func txnIterHandlerFunc(w http.ResponseWriter, lfb *block.Block) func(context.Context, datastore.CollectionEntity) (bool, error) {
 	return func(ctx context.Context, ce datastore.CollectionEntity) (bool, error) {
 		txn, ok := ce.(*transaction.Transaction)

code/go/0chain.net/miner/worker.go

@@ -291,12 +291,18 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) {
 	for {
 		logging.Logger.Debug("sync all missing nodes - loading all missing nodes...")
 		var err error
+		start := time.Now()
 		missingNodes, err = lfb.ClientState.GetAllMissingNodes()
 		if err != nil {
 			logging.Logger.Error("sync all missing nodes - get all missing nodes failed", zap.Error(err))
 			time.Sleep(3 * time.Second)
 			continue
 		}
+
+		// Record the number of missing nodes and the time it took to acquire them
+		mc.MissingNodesStat.Counter.Inc(int64(len(missingNodes)))
+		mc.MissingNodesStat.Timer.UpdateSince(start)
+
 		logging.Logger.Debug("sync all missing nodes - finish load all missing nodes",
 			zap.Int("num", len(missingNodes)))
 
@@ -317,6 +323,7 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) {
 	var (
 		batchSize = 100
 		batchs    = len(missingNodes) / batchSize
+		start     = time.Now()
 	)
 
 	for idx := 1; idx <= batchs; idx++ {
@@ -333,6 +340,8 @@ func (mc *Chain) syncAllMissingNodes(ctx context.Context) {
 		tk.Reset(2 * time.Second)
 	}
 
+	mc.MissingNodesStat.SyncTimer.UpdateSince(start)
+
 	mod := len(missingNodes) % batchSize
 	if mod > 0 {
 		wc := make(chan struct{}, 1)