Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enhance profiling and allow traces to profiles #1006

Merged
merged 3 commits into from
Jun 12, 2024
Merged

Enhance profiling and allow traces to profiles #1006

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f0e65ea
Enhance profiling and allow traces to profiles
marefr Jun 12, 2024
3d05dd2
fix spelling
marefr Jun 12, 2024
47e5f30
Apply suggestions from code review
marefr Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 35 additions & 5 deletions backend/setup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import (
"net/http"
"net/http/pprof"
"os"
"runtime"
"strconv"

pyroscopepprof "github.com/grafana/pyroscope-go/godeltaprof/http/pprof"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
Expand All @@ -23,11 +25,14 @@ const (
// PluginProfilingEnabledEnv is a constant for the GF_PLUGIN_PROFILING_ENABLED environment variable used to enable pprof.
PluginProfilingEnabledEnv = "GF_PLUGIN_PROFILING_ENABLED"

// PluginProfilerPortEnvDeprecated is a constant for the GF_PLUGINS_PROFILER_PORT environment variable use to specify a pprof port (default 6060).
// PluginProfilerPortEnvDeprecated is a constant for the GF_PLUGINS_PROFILER_PORT environment variable used to specify a pprof port (default 6060).
PluginProfilerPortEnvDeprecated = "GF_PLUGINS_PROFILER_PORT" // nolint:gosec
// PluginProfilingPortEnv is a constant for the GF_PLUGIN_PROFILING_PORT environment variable use to specify a pprof port (default 6060).
// PluginProfilingPortEnv is a constant for the GF_PLUGIN_PROFILING_PORT environment variable used to specify a pprof port (default 6060).
PluginProfilingPortEnv = "GF_PLUGIN_PROFILING_PORT" // nolint:gosec

// PluginProfilingContentionEnv is a constant for the GF_PLUGIN_PROFILING_CONTENTION environment variable used to enable contention profiling (report of all goroutine blocking and mutex contention events), default false.
marefr marked this conversation as resolved.
Show resolved Hide resolved
PluginProfilingContentionEnv = "GF_PLUGIN_PROFILING_CONTENTION" // nolint:gosec

// PluginTracingOpenTelemetryOTLPAddressEnv is a constant for the GF_INSTANCE_OTLP_ADDRESS
// environment variable used to specify the OTLP address.
PluginTracingOpenTelemetryOTLPAddressEnv = "GF_INSTANCE_OTLP_ADDRESS" // nolint:gosec
Expand Down Expand Up @@ -75,32 +80,56 @@ func setupProfiler(pluginID string) {
// compare value to plugin name
if value == pluginID {
profilerEnabled = true
Logger.Warn("Use of GF_PLUGINS_PROFILER environment variable is deprecated and not going to be supported in the future. Please use GF_PLUGIN_PROFILING_ENABLED instead.")
marefr marked this conversation as resolved.
Show resolved Hide resolved
}
} else if value, ok = os.LookupEnv(PluginProfilingEnabledEnv); ok {
if value == "true" {
profilerEnabled = true
}
}

Logger.Debug("Profiler", "enabled", profilerEnabled)
if profilerEnabled {
profilerPort := "6060"
for _, env := range []string{PluginProfilerPortEnvDeprecated, PluginProfilingPortEnv} {
if value, ok := os.LookupEnv(env); ok {
profilerPort = value

if env == PluginProfilerPortEnvDeprecated {
Logger.Warn("Use of GF_PLUGINS_PROFILER_PORT environment variable is deprecated and not going to be supported in the future. Please use GF_PLUGIN_PROFILING_PORT instead.")
marefr marked this conversation as resolved.
Show resolved Hide resolved
}

break
}
}
Logger.Info("Profiler", "port", profilerPort)

contentionProfiling := false
if value, ok := os.LookupEnv(PluginProfilingContentionEnv); ok {
if value == "true" {
contentionProfiling = true
}
}

Logger.Info("Profiling enabled", "port", profilerPort, "contentionProfilingEnabled", contentionProfiling)
portConfig := fmt.Sprintf(":%s", profilerPort)

if contentionProfiling {
runtime.SetBlockProfileRate(1)
runtime.SetMutexProfileFraction(1)
}

r := http.NewServeMux()
r.HandleFunc("/debug/pprof/", pprof.Index)
r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
r.HandleFunc("/debug/pprof/profile", pprof.Profile)
r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
r.HandleFunc("/debug/pprof/trace", pprof.Trace)

// Register godeltaprof endpoints which are more suitable for continuous profiling,
// see https://github.com/grafana/pyroscope-go/tree/main/godeltaprof.
r.HandleFunc("/debug/pprof/delta_heap", pyroscopepprof.Heap)
r.HandleFunc("/debug/pprof/delta_block", pyroscopepprof.Block)
r.HandleFunc("/debug/pprof/delta_mutex", pyroscopepprof.Mutex)

go func() {
//nolint:gosec
if err := http.ListenAndServe(portConfig, r); err != nil {
Expand Down Expand Up @@ -147,6 +176,7 @@ func SetupTracer(pluginID string, tracingOpts tracing.Opts) error {
if err != nil {
return fmt.Errorf("new trace provider: %w", err)
}

pf, err := tracerprovider.NewTextMapPropagator(tracingCfg.propagation)
if err != nil {
return fmt.Errorf("new propagator format: %w", err)
Expand All @@ -158,8 +188,8 @@ func SetupTracer(pluginID string, tracingOpts tracing.Opts) error {
}

enabled := tracingCfg.isEnabled()
Logger.Debug("Tracing", "enabled", enabled)
if enabled {
Logger.Info("Tracing enabled")
Logger.Debug(
"Tracing configuration",
"propagation", tracingCfg.propagation,
Expand Down
6 changes: 4 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,14 @@ require (
github.com/getkin/kin-openapi v0.124.0
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/grafana/otel-profiling-go v0.5.1
github.com/grafana/pyroscope-go/godeltaprof v0.1.7
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1
github.com/hashicorp/go-hclog v1.6.3
github.com/hashicorp/go-plugin v1.6.1
github.com/invopop/jsonschema v0.12.0 // for schema codgen+extraction
github.com/json-iterator/go v1.1.12
github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6
github.com/magefile/mage v1.15.0
github.com/mattetti/filebuffer v1.0.1
github.com/mitchellh/reflectwalk v1.0.2
Expand Down Expand Up @@ -74,8 +77,7 @@ require (
github.com/hashicorp/yamux v0.1.1 // indirect
github.com/invopop/yaml v0.2.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/compress v1.17.3 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand Down
16 changes: 12 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M=
github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
Expand Down Expand Up @@ -81,6 +82,10 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8=
github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls=
github.com/grafana/pyroscope-go/godeltaprof v0.1.7 h1:C11j63y7gymiW8VugJ9ZW0pWfxTZugdSJyC48olk5KY=
github.com/grafana/pyroscope-go/godeltaprof v0.1.7/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE=
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA=
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU=
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=
Expand Down Expand Up @@ -109,8 +114,8 @@ github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpR
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
Expand Down Expand Up @@ -200,8 +205,6 @@ github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3/go.mod h1:1xEUf2abjfP9
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM=
github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0=
github.com/vectordotdev/go-datemath v0.1.1-0.20220323213446-f3954d0b18ae h1:oyiy3uBj1F4O3AaFh7hUGBrJjAssJhKyAbwxtkslxqo=
github.com/vectordotdev/go-datemath v0.1.1-0.20220323213446-f3954d0b18ae/go.mod h1:PnwzbSst7KD3vpBzzlntZU5gjVa455Uqa5QPiKSYJzQ=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand All @@ -220,16 +223,20 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.26.0 h1:RH76Cl2pfOLLoCtxAPax9c
go.opentelemetry.io/contrib/propagators/jaeger v1.26.0/go.mod h1:W/cylm0ZtJK1uxsuTqoYGYPnqpZ8CeVGgW7TwfXPsGw=
go.opentelemetry.io/contrib/samplers/jaegerremote v0.20.0 h1:ja+d7Aea/9PgGxB63+E0jtRFpma717wubS0KFkZpmYw=
go.opentelemetry.io/contrib/samplers/jaegerremote v0.20.0/go.mod h1:Yc1eg51SJy7xZdOTyg1xyFcwE+ghcWh3/0hKeLo6Wlo=
go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 h1:Waw9Wfpo/IXzOI8bCB7DIk+0JZcqqsyn1JFnAc+iam8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0/go.mod h1:wnJIG4fOqyynOnnQF/eQb4/16VlX2EJAHhHgqIqWfAo=
go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8=
go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs=
go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
Expand Down Expand Up @@ -270,6 +277,7 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
27 changes: 27 additions & 0 deletions internal/tracerprovider/profiling_tracerprovider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package tracerprovider

import (
"context"

otelpyroscope "github.com/grafana/otel-profiling-go"
trace "go.opentelemetry.io/otel/trace"
)

type profilingTracerProvider struct {
trace.TracerProvider
wrappedTp TracerProvider
}

// newProfilingTracerProvider creates a new tracer provider that annotates pprof
// samples with span_id label. This allows to establish a relationship
// between pprof profiles and reported tracing spans.
func newProfilingTracerProvider(tp TracerProvider) *profilingTracerProvider {
return &profilingTracerProvider{
TracerProvider: otelpyroscope.NewTracerProvider(tp),
wrappedTp: tp,
}
}

func (tp *profilingTracerProvider) Shutdown(ctx context.Context) error {
return tp.wrappedTp.Shutdown(ctx)
}
7 changes: 6 additions & 1 deletion internal/tracerprovider/tracerprovider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,12 @@ func NewTracerProvider(address string, samplerOpts SamplerOptions, opts tracing.
if err != nil {
return nil, fmt.Errorf("new otel SamplerType: %w", err)
}
return newOtelTracerProvider(exp, sampler, opts.CustomAttributes...)
tp, err := newOtelTracerProvider(exp, sampler, opts.CustomAttributes...)
if err != nil {
return nil, err
}

return newProfilingTracerProvider(tp), nil
}

// NewTextMapPropagator takes a string-like value and returns the corresponding propagation.TextMapPropagator.
Expand Down