-
Notifications
You must be signed in to change notification settings - Fork 0
/
metrics.go
335 lines (304 loc) · 11.3 KB
/
metrics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
package cvo
import (
"time"
"github.com/prometheus/client_golang/prometheus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/cache"
configv1 "github.com/openshift/api/config/v1"
"github.com/openshift/cluster-version-operator/lib/resourcemerge"
"github.com/openshift/cluster-version-operator/pkg/internal"
)
func (optr *Operator) registerMetrics(coInformer cache.SharedInformer) error {
m := newOperatorMetrics(optr)
coInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
UpdateFunc: m.clusterOperatorChanged,
})
return prometheus.Register(m)
}
type operatorMetrics struct {
optr *Operator
conditionTransitions map[conditionKey]int
version *prometheus.GaugeVec
availableUpdates *prometheus.GaugeVec
clusterOperatorUp *prometheus.GaugeVec
clusterOperatorConditions *prometheus.GaugeVec
clusterOperatorConditionTransitions *prometheus.GaugeVec
clusterInstaller *prometheus.GaugeVec
}
func newOperatorMetrics(optr *Operator) *operatorMetrics {
return &operatorMetrics{
optr: optr,
conditionTransitions: make(map[conditionKey]int),
version: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_version",
Help: `Reports the version of the cluster in terms of seconds since
the epoch. Type 'current' is the version being applied and
the value is the creation date of the payload. The type
'desired' is returned if spec.desiredUpdate is set but the
operator has not yet updated and the value is the most
recent status transition time. The type 'failure' is set
if an error is preventing sync or upgrade with the last
transition timestamp of the condition. The type 'completed'
is the timestamp when the last image was successfully
applied. The type 'cluster' is the creation date of the
cluster version object and the current version. The type
'updating' is set when the cluster is transitioning to a
new version but has not reached the completed state and
is the time the update was started. The type 'initial' is
set to the oldest entry in the history. The from_version label
will be set to the last completed version, the initial
version for 'cluster', or empty for 'initial'.
.`,
}, []string{"type", "version", "image", "from_version"}),
availableUpdates: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_version_available_updates",
Help: "Report the count of available versions for an upstream and channel.",
}, []string{"upstream", "channel"}),
clusterOperatorUp: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_operator_up",
Help: "Reports key highlights of the active cluster operators.",
}, []string{"name", "version"}),
clusterOperatorConditions: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_operator_conditions",
Help: "Report the conditions for active cluster operators. 0 is False and 1 is True.",
}, []string{"name", "condition", "reason"}),
clusterOperatorConditionTransitions: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_operator_condition_transitions",
Help: "Reports the number of times that a condition on a cluster operator changes status",
}, []string{"name", "condition"}),
clusterInstaller: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_installer",
Help: "Reports info about the installation process and, if applicable, the install tool.",
}, []string{"type", "version", "invoker"}),
}
}
type conditionKey struct {
Name string
Type string
}
// clusterOperatorChanged detects condition transitions and records them
func (m *operatorMetrics) clusterOperatorChanged(oldObj, obj interface{}) {
oldCO, ok := oldObj.(*configv1.ClusterOperator)
if !ok {
return
}
co, ok := obj.(*configv1.ClusterOperator)
if !ok {
return
}
types := sets.NewString()
for _, older := range oldCO.Status.Conditions {
if types.Has(string(older.Type)) {
continue
}
types.Insert(string(older.Type))
newer := resourcemerge.FindOperatorStatusCondition(co.Status.Conditions, older.Type)
if newer == nil {
m.conditionTransitions[conditionKey{Name: co.Name, Type: string(older.Type)}]++
continue
}
if newer.Status != older.Status {
m.conditionTransitions[conditionKey{Name: co.Name, Type: string(older.Type)}]++
continue
}
}
for _, newer := range co.Status.Conditions {
if types.Has(string(newer.Type)) {
continue
}
types.Insert(string(newer.Type))
m.conditionTransitions[conditionKey{Name: co.Name, Type: string(newer.Type)}]++
}
}
func (m *operatorMetrics) Describe(ch chan<- *prometheus.Desc) {
ch <- m.version.WithLabelValues("", "", "", "").Desc()
ch <- m.availableUpdates.WithLabelValues("", "").Desc()
ch <- m.clusterOperatorUp.WithLabelValues("", "").Desc()
ch <- m.clusterOperatorConditions.WithLabelValues("", "", "").Desc()
ch <- m.clusterOperatorConditionTransitions.WithLabelValues("", "").Desc()
ch <- m.clusterInstaller.WithLabelValues("", "", "").Desc()
}
func (m *operatorMetrics) Collect(ch chan<- prometheus.Metric) {
current := m.optr.currentVersion()
var completed configv1.UpdateHistory
if cv, err := m.optr.cvLister.Get(m.optr.name); err == nil {
// output cluster version
var initial configv1.UpdateHistory
if last := len(cv.Status.History); last > 0 {
initial = cv.Status.History[last-1]
}
// if an update ran to completion, report the timestamp of that update and store the completed
// version for use in other metrics
for i, history := range cv.Status.History {
if history.State == configv1.CompletedUpdate {
completed = history
var previous configv1.UpdateHistory
for _, history := range cv.Status.History[i+1:] {
if history.State == configv1.CompletedUpdate {
previous = history
break
}
}
g := m.version.WithLabelValues("completed", history.Version, history.Image, previous.Version)
g.Set(float64(history.CompletionTime.Unix()))
ch <- g
break
}
}
// answers "which images were clusters initially installed with"
g := m.version.WithLabelValues("initial", initial.Version, initial.Image, "")
g.Set(float64(cv.CreationTimestamp.Unix()))
ch <- g
// answers "how old are clusters at this version currently and what version did they start at"
initialVersion := initial.Version
// clear the initial version if we have never "reached level" (i.e., we are still installing),
// which allows us to exclude clusters that are still being installed
if len(completed.Version) == 0 {
initialVersion = ""
}
g = m.version.WithLabelValues("cluster", current.Version, current.Image, initialVersion)
g.Set(float64(cv.CreationTimestamp.Unix()))
ch <- g
// answers "is there a desired update we have not yet satisfied"
if update := cv.Spec.DesiredUpdate; update != nil && update.Image != current.Image {
g = m.version.WithLabelValues("desired", update.Version, update.Image, completed.Version)
g.Set(float64(mostRecentTimestamp(cv)))
ch <- g
}
// answers "if we are failing, are we updating or reconciling"
failing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, ClusterStatusFailing)
if failing != nil && failing.Status == configv1.ConditionTrue {
if update := cv.Spec.DesiredUpdate; update != nil && update.Image != current.Image {
g = m.version.WithLabelValues("failure", update.Version, update.Image, completed.Version)
} else {
g = m.version.WithLabelValues("failure", current.Version, current.Image, completed.Version)
}
if failing.LastTransitionTime.IsZero() {
g.Set(0)
} else {
g.Set(float64(failing.LastTransitionTime.Unix()))
}
ch <- g
}
// when the CVO is transitioning towards a new version report a unique series describing it
if len(cv.Status.History) > 0 && cv.Status.History[0].State == configv1.PartialUpdate {
updating := cv.Status.History[0]
g := m.version.WithLabelValues("updating", updating.Version, updating.Image, completed.Version)
if updating.StartedTime.IsZero() {
g.Set(0)
} else {
g.Set(float64(updating.StartedTime.Unix()))
}
ch <- g
}
if len(cv.Spec.Upstream) > 0 || len(cv.Status.AvailableUpdates) > 0 || resourcemerge.IsOperatorStatusConditionTrue(cv.Status.Conditions, configv1.RetrievedUpdates) {
upstream := "<default>"
if len(cv.Spec.Upstream) > 0 {
upstream = string(cv.Spec.Upstream)
}
g := m.availableUpdates.WithLabelValues(upstream, cv.Spec.Channel)
g.Set(float64(len(cv.Status.AvailableUpdates)))
ch <- g
}
for _, condition := range cv.Status.Conditions {
if condition.Status == configv1.ConditionUnknown {
continue
}
g := m.clusterOperatorConditions.WithLabelValues("version", string(condition.Type), string(condition.Reason))
if condition.Status == configv1.ConditionTrue {
g.Set(1)
} else {
g.Set(0)
}
ch <- g
}
}
g := m.version.WithLabelValues("current", current.Version, current.Image, completed.Version)
if m.optr.releaseCreated.IsZero() {
g.Set(0)
} else {
g.Set(float64(m.optr.releaseCreated.Unix()))
}
ch <- g
// output cluster operator version and condition info
operators, _ := m.optr.coLister.List(labels.Everything())
for _, op := range operators {
// TODO: when we define how version works, report the appropriate version
var firstVersion string
for _, v := range op.Status.Versions {
firstVersion = v.Version
break
}
g := m.clusterOperatorUp.WithLabelValues(op.Name, firstVersion)
failing := resourcemerge.IsOperatorStatusConditionTrue(op.Status.Conditions, configv1.OperatorDegraded)
available := resourcemerge.IsOperatorStatusConditionTrue(op.Status.Conditions, configv1.OperatorAvailable)
if available && !failing {
g.Set(1)
} else {
g.Set(0)
}
ch <- g
for _, condition := range op.Status.Conditions {
if condition.Status == configv1.ConditionUnknown {
continue
}
g := m.clusterOperatorConditions.WithLabelValues(op.Name, string(condition.Type), string(condition.Reason))
if condition.Status == configv1.ConditionTrue {
g.Set(1)
} else {
g.Set(0)
}
ch <- g
}
}
for key, value := range m.conditionTransitions {
g := m.clusterOperatorConditionTransitions.WithLabelValues(key.Name, key.Type)
g.Set(float64(value))
ch <- g
}
installer, err := m.optr.cmConfigLister.Get(internal.InstallerConfigMap)
if err == nil {
version := "<missing>"
invoker := "<missing>"
if v, ok := installer.Data["version"]; ok {
version = v
}
if i, ok := installer.Data["invoker"]; ok {
invoker = i
}
g := m.clusterInstaller.WithLabelValues("openshift-install", version, invoker)
g.Set(1.0)
ch <- g
} else if apierrors.IsNotFound(err) {
g := m.clusterInstaller.WithLabelValues("", "", "")
g.Set(1.0)
ch <- g
}
}
// mostRecentTimestamp finds the most recent change recorded to the status and
// returns the seconds since the epoch.
func mostRecentTimestamp(cv *configv1.ClusterVersion) int64 {
var latest time.Time
if len(cv.Status.History) > 0 {
latest = cv.Status.History[0].StartedTime.Time
if t := cv.Status.History[0].CompletionTime; t != nil {
if t.Time.After(latest) {
latest = t.Time
}
}
}
for _, condition := range cv.Status.Conditions {
if condition.LastTransitionTime.After(latest) {
latest = condition.LastTransitionTime.Time
}
}
if cv.CreationTimestamp.After(latest) {
latest = cv.CreationTimestamp.Time
}
if latest.IsZero() {
return 0
}
return latest.Unix()
}