forked from grafana/otel-profiling-go
/
otelpyroscope.go
291 lines (250 loc) · 8.53 KB
/
otelpyroscope.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
package otelpyroscope
import (
"context"
"log"
"net/url"
"runtime/pprof"
"strconv"
"strings"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
const (
profileIDLabelName = "profile_id"
spanNameLabelName = "span_name"
)
var (
profileIDSpanAttributeKey = attribute.Key("pyroscope.profile.id")
profileURLSpanAttributeKey = attribute.Key("pyroscope.profile.url")
profileBaselineURLSpanAttributeKey = attribute.Key("pyroscope.profile.baseline.url")
profileDiffURLSpanAttributeKey = attribute.Key("pyroscope.profile.diff.url")
)
type Config struct {
AppName string
PyroscopeURL string
IncludeProfileURL bool
IncludeProfileBaselineURL bool
ProfileBaselineLabels map[string]string
RootOnly bool
AddSpanName bool
}
type Option func(*tracerProvider)
// WithAppName specifies the profiled application name.
// It should match the name specified in pyroscope configuration.
// Required, if profile URL or profile baseline URL is enabled.
func WithAppName(app string) Option {
return func(tp *tracerProvider) {
tp.config.AppName = app
}
}
// WithRootSpanOnly indicates that only the root span is to be profiled.
// The profile includes samples captured during child span execution
// but the spans won't have their own profiles and won't be annotated
// with pyroscope.profile attributes.
// The option is enabled by default.
func WithRootSpanOnly(x bool) Option {
return func(tp *tracerProvider) {
tp.config.RootOnly = x
}
}
// WithAddSpanName specifies whether the current span name should be added
// to the profile labels. N.B if the name is dynamic, or too many values
// are supposed, this may significantly deteriorate performance.
// By default, span name is not added to profile labels.
func WithAddSpanName(x bool) Option {
return func(tp *tracerProvider) {
tp.config.AddSpanName = x
}
}
// WithPyroscopeURL provides a base URL for the profile and baseline URLs.
// Required, if profile URL or profile baseline URL is enabled.
func WithPyroscopeURL(addr string) Option {
return func(tp *tracerProvider) {
tp.config.PyroscopeURL = addr
}
}
// WithProfileURL specifies whether to add the pyroscope.profile.url
// attribute with the URL to the span profile.
func WithProfileURL(x bool) Option {
return func(tp *tracerProvider) {
tp.config.IncludeProfileURL = x
}
}
// WithProfileBaselineURL specifies whether to add the
// pyroscope.profile.baseline.url attribute with the URL
// to the baseline profile. See WithProfileBaselineLabels.
func WithProfileBaselineURL(x bool) Option {
return func(tp *tracerProvider) {
tp.config.IncludeProfileBaselineURL = x
}
}
// WithProfileBaselineLabels provides a map of extra labels to be added to the
// baseline query alongside with pprof labels set in runtime. Typically,
// it should match the labels specified in the Pyroscope profiler config.
// Note that the map must not be modified.
func WithProfileBaselineLabels(x map[string]string) Option {
return func(tp *tracerProvider) {
tp.config.ProfileBaselineLabels = x
}
}
// WithProfileURLBuilder specifies how profile URL is to be built.
// DEPRECATED: use WithProfileURL
func WithProfileURLBuilder(b func(_ string) string) Option {
return func(tp *tracerProvider) {
tp.config.IncludeProfileURL = true
}
}
// WithDefaultProfileURLBuilder specifies the default profile URL builder.
// DEPRECATED: use WithProfileURL
func WithDefaultProfileURLBuilder(_, _ string) Option {
return func(tp *tracerProvider) {
tp.config.IncludeProfileURL = true
}
}
// tracerProvider satisfies open telemetry TracerProvider interface.
type tracerProvider struct {
tp trace.TracerProvider
config Config
}
// NewTracerProvider creates a new tracer provider that annotates pprof
// profiles with span ID tag. This allows to establish a relationship
// between pprof profiles and reported tracing spans.
func NewTracerProvider(tp trace.TracerProvider, options ...Option) trace.TracerProvider {
p := tracerProvider{
tp: tp,
config: Config{RootOnly: true},
}
for _, o := range options {
o(&p)
}
return &p
}
func (w *tracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer {
return &profileTracer{p: w, tr: w.tp.Tracer(name, opts...)}
}
type profileTracer struct {
p *tracerProvider
tr trace.Tracer
}
func (w profileTracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
isRoot := isRootSpan(trace.SpanContextFromContext(ctx))
ctx, span := w.tr.Start(ctx, spanName, opts...)
if w.p.config.RootOnly && !isRoot {
log.Println("Root Only & isn't Root")
return ctx, span
}
log.Println("profileTracer Start...")
s := spanWrapper{
Span: span,
profileID: span.SpanContext().SpanID().String(),
startTime: time.Now(),
ctx: ctx,
p: w.p,
}
// Regardless of whether a span is sampled ot not, if configured,
// span_name pprof tag should be set to ensure profile consistency.
labels := make([]string, 0, 4)
if w.p.config.AddSpanName && spanName != "" {
labels = append(labels, spanNameLabelName, spanName)
}
if span.SpanContext().IsSampled() && s.profileID != "" {
// Set profile_id pprof tag only if the span is sampled.
log.Println("======================================================")
log.Println(" Set profile_id pprof tag only if the span is sampled.")
log.Println("profileID: ", s.profileID)
log.Println("======================================================")
labels = append(labels, profileIDLabelName, s.profileID)
}
log.Println("Labels, ", labels)
log.Println("pprof.Labels, ", pprof.Labels(labels...))
ctx = pprof.WithLabels(ctx, pprof.Labels(labels...))
log.Printf(
"span_name=%v, profile_id=%v",
ctx.Value("span_name"), ctx.Value("profile_id"),
)
pprof.SetGoroutineLabels(ctx)
s.pprofCtx = ctx
return ctx, &s
}
var emptySpanID trace.SpanID
func isRootSpan(s trace.SpanContext) bool {
return s.IsRemote() || s.SpanID() == emptySpanID
}
type spanWrapper struct {
trace.Span
// Span context.
ctx context.Context
// Current pprof context with labels.
pprofCtx context.Context
profileID string
startTime time.Time
p *tracerProvider
}
func (s spanWrapper) End(options ...trace.SpanEndOption) {
// By this profiles can be easily associated with the corresponding spans.
// We use span ID as a profile ID because it perfectly fits profiling scope.
// In practice, a profile ID is an arbitrary string identifying the execution
// scope that is associated with a tracing span.
s.SetAttributes(profileIDSpanAttributeKey.String(s.profileID))
// Optionally specify the profile URL.
if s.p.config.IncludeProfileURL {
s.setProfileURL()
}
if s.p.config.IncludeProfileBaselineURL {
s.setBaselineURLs()
}
s.Span.End(options...)
// pprof.SetGoroutineLabels(s.ctx)
}
func (s spanWrapper) setProfileURL() {
q := make(url.Values, 3)
from := strconv.FormatInt(s.startTime.UnixNano(), 10)
until := strconv.FormatInt(time.Now().UnixNano(), 10)
q.Set("query", s.p.config.AppName+`.cpu{`+profileIDLabelName+`="`+s.profileID+`"}`)
q.Set("from", from)
q.Set("until", until)
s.SetAttributes(profileURLSpanAttributeKey.String(s.p.config.PyroscopeURL + "/?" + q.Encode()))
}
func (s spanWrapper) setBaselineURLs() {
var b strings.Builder
pprof.ForLabels(s.pprofCtx, func(key, value string) bool {
if key == profileIDLabelName {
return true
}
if s.p.config.ProfileBaselineLabels != nil {
if _, ok := s.p.config.ProfileBaselineLabels[key]; ok {
return true
}
}
writeLabel(&b, key, value)
return true
})
for key, value := range s.p.config.ProfileBaselineLabels {
if value != "" {
writeLabel(&b, key, value)
}
}
q := make(url.Values, 9)
baselineQuery := s.p.config.AppName + `.cpu{` + b.String() + `}`
baselineFrom := strconv.FormatInt(s.startTime.Add(-time.Hour).UnixNano(), 10)
until := strconv.FormatInt(time.Now().UnixNano(), 10)
q.Set("query", baselineQuery)
q.Set("from", baselineFrom)
q.Set("until", until)
q.Set("rightQuery", s.p.config.AppName+`.cpu{`+profileIDLabelName+`="`+s.profileID+`"}`)
q.Set("rightFrom", strconv.FormatInt(s.startTime.UnixNano(), 10))
q.Set("rightUntil", until)
q.Set("leftQuery", baselineQuery)
q.Set("leftFrom", baselineFrom)
q.Set("leftUntil", until)
qs := q.Encode()
s.SetAttributes(profileBaselineURLSpanAttributeKey.String(s.p.config.PyroscopeURL + "/comparison?" + qs))
s.SetAttributes(profileDiffURLSpanAttributeKey.String(s.p.config.PyroscopeURL + "/comparison-diff?" + qs))
}
func writeLabel(b *strings.Builder, k, v string) {
if b.Len() > 0 {
b.WriteByte(',')
}
b.WriteString(k + `="` + v + `"`)
}