chore: add support for profiling (#3704)

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/goware/urlx v0.3.2
+	github.com/grafana/pyroscope-go v1.1.1
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/j2gg0s/otsql v0.14.0
 	github.com/jackc/pgx/v5 v5.4.2
@@ -123,6 +124,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/gookit/color v1.5.4 // indirect
+	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
@@ -138,7 +140,7 @@ require (
 	github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/knadh/koanf v1.5.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lib/pq v1.10.5 // indirect

go.sum

@@ -992,6 +992,10 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/goware/urlx v0.3.2 h1:gdoo4kBHlkqZNaf6XlQ12LGtQOmpKJrR04Rc3RnpJEo=
 github.com/goware/urlx v0.3.2/go.mod h1:h8uwbJy68o+tQXCGZNa9D73WN8n0r9OBae5bUnLcgjw=
+github.com/grafana/pyroscope-go v1.1.1 h1:PQoUU9oWtO3ve/fgIiklYuGilvsm8qaGhlY4Vw6MAcQ=
+github.com/grafana/pyroscope-go v1.1.1/go.mod h1:Mw26jU7jsL/KStNSGGuuVYdUq7Qghem5P8aXYXSXG88=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.6/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -1239,8 +1243,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY
 github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
+github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio=
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=

server/app/app.go

@@ -142,7 +142,7 @@ func (app *App) initAnalytics(configFromDB config.Config) error {
 var instanceID = id.GenerateID().String()
 
 func (app *App) Start(opts ...appOption) error {
-	// instanceID is a temprary ID for this instance of the server
+	// instanceID is a temporary ID for this instance of the server
 	// it is regenerated on every start intentionally
 	for _, opt := range opts {
 		opt(app)

server/cmd/serve.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/kubeshop/tracetest/server/app"
 	"github.com/kubeshop/tracetest/server/config"
+	"github.com/kubeshop/tracetest/server/telemetry"
 	"github.com/spf13/cobra"
 )
 
@@ -36,6 +37,11 @@ var serveCmd = &cobra.Command{
 			os.Exit(1)
 		}()
 
+		profilerConfig := cfg.ApplicationProfiler()
+		if profilerConfig.Enabled {
+			telemetry.StartProfiler(profilerConfig.Name, profilerConfig.Endpoint, profilerConfig.SamplingRate)
+		}
+
 		wg.Add(1)
 		err := appInstance.Start(app.WithProvisioningFile(provisioningFile))
 		if err != nil {

server/config/exporters.go

@@ -17,7 +17,8 @@ type (
 	}
 
 	telemetry struct {
-		Exporters map[string]TelemetryExporterOption `yaml:",omitempty" mapstructure:"exporters"`
+		ApplicationProfiler ApplicationProfiler                `yaml:",omitempty" mapstructure:"profiler"`
+		Exporters           map[string]TelemetryExporterOption `yaml:",omitempty" mapstructure:"exporters"`
 	}
 
 	TelemetryExporterOption struct {
@@ -35,8 +36,22 @@ type (
 	OTELCollectorConfig struct {
 		Endpoint string `yaml:",omitempty" mapstructure:"endpoint"`
 	}
+
+	ApplicationProfiler struct {
+		Name         string `yaml:",omitempty" mapstructure:"name"`
+		Enabled      bool   `yaml:",omitempty" mapstructure:"enabled"`
+		Endpoint     string `yaml:",omitempty" mapstructure:"endpoint"`
+		SamplingRate int    `yaml:",omitempty" mapstructure:"samplingRate"`
+	}
 )
 
+func (c *AppConfig) ApplicationProfiler() *ApplicationProfiler {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	return &c.config.Telemetry.ApplicationProfiler
+}
+
 func (c *AppConfig) Exporter() (*TelemetryExporterOption, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()

server/telemetry/profiler.go

@@ -0,0 +1,51 @@
+package telemetry
+
+import (
+	"os"
+	"runtime"
+
+	pyroscope "github.com/grafana/pyroscope-go"
+)
+
+const (
+	MutexProfileFractionRate = 5 // it samples 1/5 of data
+	BlockProfileFractionRate = 5
+)
+
+func StartProfiler(applicationName, telemetryServer string, samplingPercentage int) {
+	samplingRate := 100 / samplingPercentage
+
+	runtime.SetMutexProfileFraction(samplingRate)
+	runtime.SetBlockProfileRate(samplingRate)
+
+	pyroscope.Start(pyroscope.Config{
+		ApplicationName: applicationName,
+
+		// replace this with the address of pyroscope server
+		ServerAddress: telemetryServer,
+
+		// you can disable logging by setting this to nil
+		Logger: pyroscope.StandardLogger,
+
+		// you can provide static tags via a map:
+		Tags: map[string]string{
+			"hostname": os.Getenv("HOSTNAME"),
+		},
+
+		ProfileTypes: []pyroscope.ProfileType{
+			// these profile types are enabled by default:
+			pyroscope.ProfileCPU,
+			pyroscope.ProfileAllocObjects,
+			pyroscope.ProfileAllocSpace,
+			pyroscope.ProfileInuseObjects,
+			pyroscope.ProfileInuseSpace,
+
+			// these profile types are optional:
+			pyroscope.ProfileGoroutines,
+			pyroscope.ProfileMutexCount,
+			pyroscope.ProfileMutexDuration,
+			pyroscope.ProfileBlockCount,
+			pyroscope.ProfileBlockDuration,
+		},
+	})
+}