-
-
Notifications
You must be signed in to change notification settings - Fork 74
/
scripts.go
168 lines (155 loc) · 5.59 KB
/
scripts.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
package exporter
import (
"bytes"
"context"
"fmt"
"net/http"
"os"
"os/exec"
"strconv"
"strings"
"time"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/prometheus/client_golang/prometheus"
)
// runScript runs a program with some arguments; the program is
// args[0]. The timeout argument is in seconds, and if it's larger
// than zero, it is exported into the environment as $SCRIPT_TIMEOUT
// (its raw value) and $SCRIPT_DEADLINE, which is the Unix timestamp
// (including fractional parts) when the deadline will expire. If
// enforced is true, the timeout will be enforced by script_exporter,
// by killing the script if the timeout is reached, and
// $SCRIPT_TIMEOUT_ENFORCED will be set to 1 in the environment to
// inform the script of this.
//
// Note that killing the script is only a best effort attempt to
// terminate its execution and time out the request. Sub-processes may
// not be terminated, and termination may not be entirely successful.
//
// Tentatively, we do not inherit the context from the HTTP request.
// Doing so would provide automatic termination should the client
// close the connection, but it would mean that all scripts would
// be subject to abrupt termination regardless of any 'enforced:'
// settings. Right now, abrupt termination requires opting in in
// the configuration file.
func runScript(name string, logger log.Logger, timeout float64, enforced bool, args []string, env map[string]string) (string, int, error) {
// We go through a great deal of work to get a deadline with
// fractional seconds that we can expose in an environment
// variable. However, this is pretty much necessary since
// we've copied Blackbox's default of a half second adjustment
// to the raw Prometheus timeout. We can hardly do that and
// then round our deadlines (or our raw timeouts) off to full
// seconds.
ns := float64(time.Second)
deadline := time.Now().Add(time.Duration(timeout * ns))
dlfractional := float64(deadline.UnixNano()) / ns
var cmd *exec.Cmd
var cancel context.CancelFunc
ctx := context.Background()
if timeout > 0 && enforced {
ctx, cancel = context.WithDeadline(context.Background(), deadline)
defer cancel()
}
//nolint:gosec
cmd = exec.CommandContext(ctx, args[0], args[1:]...)
// Set environments variables
cmd.Env = os.Environ()
for key, value := range env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, value))
}
if timeout > 0 {
// Three digits of fractional precision in the seconds and
// the deadline are probably excessive, given that we're
// running external programs. But better slightly excessive
// than not enough precision.
cmd.Env = append(cmd.Env, fmt.Sprintf("SCRIPT_TIMEOUT=%0.3f", timeout))
cmd.Env = append(cmd.Env, fmt.Sprintf("SCRIPT_DEADLINE=%0.3f", dlfractional))
var ienforced int
if enforced {
ienforced = 1
}
cmd.Env = append(cmd.Env, fmt.Sprintf("SCRIPT_TIMEOUT_ENFORCED=%d", ienforced))
}
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
level.Error(logger).Log(
"msg", fmt.Sprintf("Script '%s' execution failed", name),
"cmd", strings.Join(args, " "),
"stdout", stdout.String(),
"stderr", stderr.String(),
"env", strings.Join(cmd.Env, " "),
"err", err,
)
if exitError, ok := err.(*exec.ExitError); ok {
return stdout.String(), exitError.ExitCode(), err
}
return stdout.String(), -1, err
}
level.Debug(logger).Log(
"msg", fmt.Sprintf("Script '%s' execution succeed", name),
"cmd", strings.Join(args, " "),
"stdout", stdout.String(),
"stderr", stderr.String(),
"env", strings.Join(cmd.Env, " "),
)
return stdout.String(), 0, nil
}
// getTimeout gets the Prometheus scrape timeout (in seconds) from the
// HTTP request, either from a 'timeout' query parameter or from the
// special HTTP header that Prometheus inserts on scrapes, and returns
// it. If there is a timeout, it is modified down by the offset.
//
// If the there is an error or no timeout is specified, it returns
// the maxTimeout configured for the script (the default value for this
// is 0, which means no timeout)
func getTimeout(r *http.Request, offset float64, maxTimeout float64) float64 {
v := r.URL.Query().Get("timeout")
if v == "" {
v = r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds")
}
if v == "" {
return maxTimeout
}
ts, err := strconv.ParseFloat(v, 64)
adjusted := ts - offset
switch {
case err != nil:
return maxTimeout
case maxTimeout < adjusted && maxTimeout > 0:
return maxTimeout
case adjusted <= 0:
return 0
default:
return adjusted
}
}
// instrumentScript wraps the underlying http.Handler with Prometheus
// instrumentation to produce per-script metrics on the number of
// requests in flight, the number of requests in total, and the
// distribution of their duration. Requests without a 'script=' query
// parameter are not instrumented (and will probably be rejected).
func instrumentScript(obs prometheus.ObserverVec, cnt *prometheus.CounterVec, g *prometheus.GaugeVec, next http.Handler) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
sn := r.URL.Query().Get("script")
if sn == "" {
// Rather than make up a fake script label, such
// as "NONE", we let the request fall through without
// instrumenting it. Under normal circumstances it
// will fail anyway, as metricsHandler() will
// reject it.
next.ServeHTTP(w, r)
return
}
labels := prometheus.Labels{"script": sn}
g.With(labels).Inc()
defer g.With(labels).Dec()
now := time.Now()
next.ServeHTTP(w, r)
obs.With(labels).Observe(time.Since(now).Seconds())
cnt.With(labels).Inc()
})
}