forked from iotexproject/iotex-core
/
heartbeat.go
137 lines (118 loc) · 4.43 KB
/
heartbeat.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
// Copyright (c) 2018 IoTeX
// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no
// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent
// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache
// License 2.0 that can be found in the LICENSE file.
package itx
import (
"context"
"encoding/json"
"strconv"
"github.com/iotexproject/go-fsm"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
"github.com/iotexproject/iotex-core/consensus"
"github.com/iotexproject/iotex-core/consensus/scheme/rolldpos"
"github.com/iotexproject/iotex-core/dispatcher"
"github.com/iotexproject/iotex-core/pkg/log"
"github.com/iotexproject/iotex-core/pkg/version"
)
// TODO: HeartbeatHandler opens encapsulation of a few structs to inspect the internal status, we need to find a better
// approach to do so in the future
var heartbeatMtc = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "iotex_heartbeat_status",
Help: "Node heartbeat status.",
},
[]string{"status_type", "source"},
)
var versionMtc = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "iotex_version_status",
Help: "Node software version status.",
},
[]string{"type", "value"},
)
func init() {
prometheus.MustRegister(heartbeatMtc)
prometheus.MustRegister(versionMtc)
}
// HeartbeatHandler is the handler to periodically log the system key metrics
type HeartbeatHandler struct {
s *Server
}
// NewHeartbeatHandler instantiates a HeartbeatHandler instance
func NewHeartbeatHandler(s *Server) *HeartbeatHandler {
return &HeartbeatHandler{s: s}
}
// Log executes the logging logic
func (h *HeartbeatHandler) Log() {
// Network metrics
p2pAgent := h.s.P2PAgent()
// Dispatcher metrics
dp, ok := h.s.Dispatcher().(*dispatcher.IotxDispatcher)
if !ok {
log.L().Error("dispatcher is not the instance of IotxDispatcher")
return
}
numDPEvts := len(*dp.EventChan())
dpEvtsAudit, err := json.Marshal(dp.EventAudit())
if err != nil {
log.L().Error("error when serializing the dispatcher event audit map.", zap.Error(err))
return
}
ctx := context.Background()
peers, err := p2pAgent.Neighbors(ctx)
if err != nil {
log.L().Debug("error when get neighbors.", zap.Error(err))
peers = nil
}
numPeers := len(peers)
log.L().Info("Node status.",
zap.Int("numPeers", numPeers),
zap.Int("pendingDispatcherEvents", numDPEvts),
zap.String("pendingDispatcherEventsAudit", string(dpEvtsAudit)))
heartbeatMtc.WithLabelValues("numPeers", "node").Set(float64(numPeers))
heartbeatMtc.WithLabelValues("pendingDispatcherEvents", "node").Set(float64(numDPEvts))
// chain service
for _, c := range h.s.chainservices {
// Consensus metrics
cs, ok := c.Consensus().(*consensus.IotxConsensus)
if !ok {
log.L().Info("consensus is not the instance of IotxConsensus.")
return
}
rolldpos, ok := cs.Scheme().(*rolldpos.RollDPoS)
numPendingEvts := 0
var state fsm.State
if ok {
numPendingEvts = rolldpos.NumPendingEvts()
state = rolldpos.CurrentState()
} else {
log.L().Debug("scheme is not the instance of RollDPoS")
}
// Block metrics
height := c.Blockchain().TipHeight()
actPoolSize := c.ActionPool().GetSize()
actPoolCapacity := c.ActionPool().GetCapacity()
targetHeight := c.BlockSync().TargetHeight()
log.L().Info("chain service status",
zap.Int("rolldposEvents", numPendingEvts),
zap.String("fsmState", string(state)),
zap.Uint64("blockchainHeight", height),
zap.Uint64("actpoolSize", actPoolSize),
zap.Uint64("actpoolCapacity", actPoolCapacity),
zap.Uint32("chainID", c.ChainID()),
zap.Uint64("targetHeight", targetHeight),
)
chainIDStr := strconv.FormatUint(uint64(c.ChainID()), 10)
heartbeatMtc.WithLabelValues("pendingRolldposEvents", chainIDStr).Set(float64(numPendingEvts))
heartbeatMtc.WithLabelValues("blockchainHeight", chainIDStr).Set(float64(height))
heartbeatMtc.WithLabelValues("actpoolSize", chainIDStr).Set(float64(actPoolSize))
heartbeatMtc.WithLabelValues("actpoolCapacity", chainIDStr).Set(float64(actPoolCapacity))
heartbeatMtc.WithLabelValues("targetHeight", chainIDStr).Set(float64(targetHeight))
heartbeatMtc.WithLabelValues("packageVersion", version.PackageVersion).Set(1)
heartbeatMtc.WithLabelValues("packageCommitID", version.PackageCommitID).Set(1)
heartbeatMtc.WithLabelValues("goVersion", version.GoVersion).Set(1)
}
}