-
Notifications
You must be signed in to change notification settings - Fork 0
/
metrics.go
128 lines (106 loc) · 3.87 KB
/
metrics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
package main
import (
"context"
"fmt"
"net"
"net/http"
"os"
"os/signal"
"strconv"
"syscall"
"time"
"github.com/monzo/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
const (
listenHostPortEnv = "FOREX_EXPORTER_LISTEN"
defaultListenHost = "0.0.0.0"
defaultListenPort = 9299
recentRestartsWindow = time.Minute * 10
maxRecentRestarts = 5
terminationGraceSeconds = 10
)
var (
lastRestartTime time.Time
recentRestarts int
)
var (
exchangeRateMetrics = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "forex_exporter",
Name: "exchange_rate",
Help: "Record the exchange rate between a currency pair",
}, []string{"source_currency", "target_currency", "data_source"})
exchangeRateTimestampMetrics = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "forex_exporter",
Name: "exchange_rate_timestamp",
Help: "The UNIX timestamp when the exchange rate between a currency pair was last published",
}, []string{"source_currency", "target_currency", "data_source"})
)
func registerForexRate(source_currency, targetCurrency, dataSource string, rate float64) {
exchangeRateMetrics.WithLabelValues(source_currency, targetCurrency, dataSource).Set(rate)
}
// This timestamp is supplied by the data source for when the forex rate was last published,
// as forex markets generally close on the weekend and at certain other times.
func registerForexRateTimestamp(source_currency, targetCurrency, dataSource string, timestamp int) {
exchangeRateTimestampMetrics.WithLabelValues(source_currency, targetCurrency, dataSource).Set(float64(timestamp))
}
func startMetricsServer(errChan chan error) error {
ctx := context.Background()
host := defaultListenHost
port := defaultListenPort
envHostPort := os.Getenv(listenHostPortEnv)
if envHostPort != "" {
parsedHost, parsedPort, err := net.SplitHostPort(envHostPort)
if err != nil {
slog.Critical(ctx, "Invalid listen host port: %s, cannot initialize", envHostPort)
return err
}
host = parsedHost
portNum, err := strconv.ParseInt(parsedPort, 10, 64)
if err != nil || portNum < 1 || portNum > 32767 {
slog.Critical(ctx, "Invalid port: %s, cannot initialize", parsedPort)
return err
}
port = int(portNum)
}
srvmx := http.NewServeMux()
// net.SplitHostPort accepts unspecified host, which means when e.g. ":8080" is
// requested the host will simply be empty string and is valid.
server := &http.Server{Addr: fmt.Sprintf("%s:%d", host, port), Handler: srvmx}
srvmx.Handle("/metrics", promhttp.Handler())
// A simple automatic recovery routine for the metrics server with limited recent retries
go func() {
for {
if err := server.ListenAndServe(); err != nil {
slog.Error(ctx, "Local metrics server encountered error: %v", err)
timeOfError := time.Now()
if timeOfError.Sub(lastRestartTime) > recentRestartsWindow {
recentRestarts = 0
}
if recentRestarts > maxRecentRestarts {
slog.Critical(ctx, "Too many recent restarts (%d), exiting.", maxRecentRestarts)
errChan <- err
return
}
slog.Warn(ctx, "Restaring metrics server following recent error %v", err)
recentRestarts++
}
}
}()
// Gracefully stop if terminated
go func() {
done := make(chan os.Signal, 1)
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
<-done
slog.Info(ctx, "Shutting down metrics server with grace period %ds...", terminationGraceSeconds)
ctx, cancel := context.WithTimeout(context.Background(), terminationGraceSeconds*time.Second)
defer cancel()
if err := server.Shutdown(ctx); err != nil {
slog.Error(ctx, "Error shutting down server: %+v, bailing out", err)
return
}
}()
return nil
}