/
metrics.go
118 lines (105 loc) · 3.5 KB
/
metrics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
package metrics
import (
"fmt"
"net/http"
"time"
api "github.com/SAP/stewardci-core/pkg/apis/steward/v1alpha1"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
klog "k8s.io/klog/v2"
)
//TODO: Move to pipeline run controller
// Metrics provides metrics
type Metrics interface {
CountStart()
CountResult(api.Result)
ObserveDurationByState(state *api.StateItem) error
ObserveUpdateDurationByType(kind string, duration time.Duration)
StartServer()
SetQueueCount(int)
}
type metrics struct {
Started prometheus.Counter
Completed *prometheus.CounterVec
Duration *prometheus.HistogramVec
Update *prometheus.HistogramVec
Queued prometheus.Gauge
Total prometheus.Gauge
}
// NewMetrics create metrics
func NewMetrics() Metrics {
return &metrics{
Started: prometheus.NewCounter(prometheus.CounterOpts{
Name: "steward_pipelineruns_started_total",
Help: "total number of started pipelines",
}),
Completed: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "steward_pipelineruns_completed_total",
Help: "completed pipelines",
},
[]string{"result"}),
Duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "steward_pipelinerun_duration_seconds",
Help: "pipeline run durations",
Buckets: prometheus.ExponentialBuckets(0.125, 2, 15),
},
[]string{"state"}),
Queued: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "steward_queued_total",
Help: "total queue count of pipelineruns",
}),
Update: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "steward_pipelinerun_update_seconds",
Help: "pipeline run update duration",
Buckets: prometheus.ExponentialBuckets(0.001, 1.3, 30),
},
[]string{"type"}),
Total: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "steward_pipelineruns_total",
Help: "total number of pipelineruns",
}),
}
}
// StartServer registers metrics and start http listener
func (metrics *metrics) StartServer() {
prometheus.MustRegister(metrics.Started)
prometheus.MustRegister(metrics.Completed)
prometheus.MustRegister(metrics.Duration)
prometheus.MustRegister(metrics.Update)
prometheus.MustRegister(metrics.Queued)
go provideMetrics()
}
func provideMetrics() {
http.Handle("/metrics", promhttp.Handler())
err := http.ListenAndServe(":9090", nil)
if err != nil {
klog.Fatalf("Failed to start metrics server for pipeline run controller:%v", err)
}
}
// CountStart counts the start events
func (metrics *metrics) CountStart() {
metrics.Started.Inc()
}
// CountResult counts the completed events by result type
func (metrics *metrics) CountResult(result api.Result) {
metrics.Completed.With(prometheus.Labels{"result": string(result)}).Inc()
}
// ObserveDurationByState logs duration of the state
func (metrics *metrics) ObserveDurationByState(state *api.StateItem) error {
if state.StartedAt.IsZero() {
return fmt.Errorf("cannot observe StateItem if StartedAt is not set")
}
duration := state.FinishedAt.Sub(state.StartedAt.Time)
if duration < 0 {
return fmt.Errorf("cannot observe StateItem if FinishedAt is before StartedAt")
}
metrics.Duration.With(prometheus.Labels{"state": string(state.State)}).Observe(duration.Seconds())
return nil
}
func (metrics *metrics) ObserveUpdateDurationByType(typ string, duration time.Duration) {
metrics.Update.With(prometheus.Labels{"type": typ}).Observe(duration.Seconds())
}
// SetQueueCount logs queue count metric
func (metrics *metrics) SetQueueCount(count int) {
metrics.Queued.Set(float64(count))
}