/
metrics.go
99 lines (90 loc) · 2.6 KB
/
metrics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
package metrics
import (
"context"
"github.com/prometheus/client_golang/prometheus"
appsv1 "k8s.io/api/apps/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)
// Metrics includes metrics used in notebook controller
type Metrics struct {
cli client.Client
runningNotebooks *prometheus.GaugeVec
NotebookCreation *prometheus.CounterVec
NotebookFailCreation *prometheus.CounterVec
NotebookCullingCount *prometheus.CounterVec
NotebookCullingTimestamp *prometheus.GaugeVec
}
func NewMetrics(cli client.Client) *Metrics {
m := &Metrics{
cli: cli,
runningNotebooks: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "notebook_running",
Help: "Current running notebooks in the cluster",
},
[]string{"namespace"},
),
NotebookCreation: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "notebook_create_total",
Help: "Total times of creating notebooks",
},
[]string{"namespace"},
),
NotebookFailCreation: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "notebook_create_failed_total",
Help: "Total failure times of creating notebooks",
},
[]string{"namespace"},
),
NotebookCullingCount: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "notebook_culling_total",
Help: "Total times of culling notebooks",
},
[]string{"namespace", "name"},
),
NotebookCullingTimestamp: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "last_notebook_culling_timestamp_seconds",
Help: "Timestamp of the last notebook culling in seconds",
},
[]string{"namespace", "name"},
),
}
metrics.Registry.MustRegister(m)
return m
}
// Describe implements the prometheus.Collector interface.
func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
m.runningNotebooks.Describe(ch)
m.NotebookCreation.Describe(ch)
m.NotebookFailCreation.Describe(ch)
}
// Collect implements the prometheus.Collector interface.
func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
m.scrape()
m.runningNotebooks.Collect(ch)
m.NotebookCreation.Collect(ch)
m.NotebookFailCreation.Collect(ch)
}
// scrape gets current running notebook statefulsets.
func (m *Metrics) scrape() {
stsList := &appsv1.StatefulSetList{}
err := m.cli.List(context.TODO(), stsList)
if err != nil {
return
}
stsCache := make(map[string]float64)
for _, v := range stsList.Items {
name, ok := v.Spec.Template.GetLabels()["notebook-name"]
if ok && name == v.Name {
stsCache[v.Namespace] += 1
}
}
for ns, v := range stsCache {
m.runningNotebooks.WithLabelValues(ns).Set(v)
}
}