-
Notifications
You must be signed in to change notification settings - Fork 46
/
0057_watch_metrics_prometheus_ignore.txt
160 lines (142 loc) · 5.98 KB
/
0057_watch_metrics_prometheus_ignore.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
exec bash -x ./prometheus.sh &
exec bash -c 'I=0 ; while [ ! -f prometheus.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done'
exec bash -x ./test.sh &
pint.ok watch --listen=:6057 --pidfile=pint.pid rules
cmp curl.txt metrics.txt
-- test.sh --
sleep 3
curl -s http://127.0.0.1:6057/metrics | grep 'pint_' | perl -pe "s/^([a-zA-Z].+)[ ]([0-9\.\-\+eE]+)$/\1/g" > curl.txt
cat pint.pid | xargs kill
cat prometheus.pid | xargs kill
-- rules/1.yml --
- record: broken
expr: foo / count())
- record: aggregate
expr: sum(foo) without(job)
- alert: comparison
expr: foo
-- .pint.hcl --
prometheus "prom1" {
uri = "http://127.0.0.1:7057"
timeout = "5s"
required = false
}
prometheus "prom2" {
uri = "http://127.0.0.1:1057"
timeout = "5s"
required = false
}
-- metrics.txt --
# HELP pint_check_duration_seconds How long did a check took to complete
# TYPE pint_check_duration_seconds summary
pint_check_duration_seconds_sum{check="alerts/comparison"}
pint_check_duration_seconds_count{check="alerts/comparison"}
pint_check_duration_seconds_sum{check="alerts/for"}
pint_check_duration_seconds_count{check="alerts/for"}
pint_check_duration_seconds_sum{check="alerts/template"}
pint_check_duration_seconds_count{check="alerts/template"}
pint_check_duration_seconds_sum{check="promql/fragile"}
pint_check_duration_seconds_count{check="promql/fragile"}
pint_check_duration_seconds_sum{check="promql/rate"}
pint_check_duration_seconds_count{check="promql/rate"}
pint_check_duration_seconds_sum{check="promql/regexp"}
pint_check_duration_seconds_count{check="promql/regexp"}
pint_check_duration_seconds_sum{check="promql/series"}
pint_check_duration_seconds_count{check="promql/series"}
pint_check_duration_seconds_sum{check="promql/syntax"}
pint_check_duration_seconds_count{check="promql/syntax"}
pint_check_duration_seconds_sum{check="promql/vector_matching"}
pint_check_duration_seconds_count{check="promql/vector_matching"}
# HELP pint_check_iterations_total Total number of completed check iterations since pint start
# TYPE pint_check_iterations_total counter
pint_check_iterations_total
# HELP pint_last_run_time_seconds Last checks run completion time since unix epoch in seconds
# TYPE pint_last_run_time_seconds gauge
pint_last_run_time_seconds
# HELP pint_problem Prometheus rule problem reported by pint
# TYPE pint_problem gauge
pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="query using \"prom1\" on http://127.0.0.1:7057 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"}
pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="query using \"prom1\" on http://127.0.0.1:7057 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"}
pint_problem{filename="rules/1.yml",kind="recording",name="broken",problem="syntax error: no arguments for aggregate expression provided",reporter="promql/syntax",severity="fatal"}
# HELP pint_problems Total number of problems reported by pint
# TYPE pint_problems gauge
pint_problems
# HELP pint_prometheus_queries_total Total number of all prometheus queries
# TYPE pint_prometheus_queries_total counter
pint_prometheus_queries_total{endpoint="/api/v1/query",name="prom1"}
pint_prometheus_queries_total{endpoint="/api/v1/query",name="prom2"}
pint_prometheus_queries_total{endpoint="/api/v1/status/config",name="prom1"}
pint_prometheus_queries_total{endpoint="/api/v1/status/config",name="prom2"}
# HELP pint_prometheus_query_errors_total Total number of failed prometheus queries
# TYPE pint_prometheus_query_errors_total counter
pint_prometheus_query_errors_total{endpoint="/api/v1/query",name="prom1",reason="api/bad_response"}
pint_prometheus_query_errors_total{endpoint="/api/v1/query",name="prom2",reason="connection/error"}
pint_prometheus_query_errors_total{endpoint="/api/v1/status/config",name="prom1",reason="api/server_error"}
pint_prometheus_query_errors_total{endpoint="/api/v1/status/config",name="prom2",reason="connection/error"}
# HELP pint_rules_parsed_total Total number of rules parsed since startup
# TYPE pint_rules_parsed_total counter
pint_rules_parsed_total{kind="alerting"}
pint_rules_parsed_total{kind="invalid"}
pint_rules_parsed_total{kind="recording"}
# HELP pint_version Version information
# TYPE pint_version gauge
pint_version{version="unknown"}
-- prometheus.go --
package main
import (
"context"
"io"
"log"
"net"
"net/http"
"os"
"os/signal"
"strconv"
"syscall"
"time"
)
func main() {
http.HandleFunc("/api/v1/status/config", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(500)
time.Sleep(time.Millisecond * 100)
io.WriteString(w, "Fatal error")
})
http.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(400)
time.Sleep(time.Millisecond * 200)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"status":"error",
"errorType":"bad_data",
"error":"bogus query"
}`))
io.WriteString(w, "Fatal error")
})
listener, err := net.Listen("tcp", "127.0.0.1:7057")
if err != nil {
log.Fatal(err)
}
server := &http.Server{
Addr: "127.0.0.1:7057",
}
go func() {
_ = server.Serve(listener)
}()
pid := os.Getpid()
err = os.WriteFile("prometheus.pid", []byte(strconv.Itoa(pid)), 0644)
if err != nil {
log.Fatal(err)
}
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
go func() {
time.Sleep(time.Minute*2)
stop <- syscall.SIGTERM
}()
<-stop
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
server.Shutdown(ctx)
}
-- prometheus.sh --
env GOCACHE=$TMPDIR go run prometheus.go