Fix memory leak caused by otelhttp (#38844)

* Fix memory leak caused by otelhttp * go mod tidy * go mod tidy
gravitational · Mar 4, 2024 · 59cf8b5 · 59cf8b5
1 parent 10e336c
commit 59cf8b5
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 9 deletions.
diff --git a/api/go.mod b/api/go.mod
@@ -19,6 +19,7 @@ require (
 	go.opentelemetry.io/otel v1.24.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0
+	go.opentelemetry.io/otel/metric v1.24.0
 	go.opentelemetry.io/otel/sdk v1.24.0
 	go.opentelemetry.io/otel/trace v1.24.0
 	go.opentelemetry.io/proto/otlp v1.1.0
@@ -46,7 +47,6 @@ require (
 	github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russellhaering/goxmldsig v1.4.0 // indirect
-	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c // indirect

diff --git a/api/observability/tracing/http/http.go b/api/observability/tracing/http/http.go
@@ -18,8 +18,20 @@ import (
 	"net/http"
 
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+	metricnoop "go.opentelemetry.io/otel/metric/noop"
 )
 
+func init() {
+	// There's a memory leak in otelhttp caused by the default global
+	// delegating meter provider. Here we set the no op meter provider to
+	// avoid this.
+	//
+	// See the upstream issue for more:
+	// https://github.com/open-telemetry/opentelemetry-go-contrib/issues/5190
+	otel.SetMeterProvider(metricnoop.MeterProvider{})
+}
+
 // TransportFormatter is a span formatter that may be provided to
 // [otelhttp.WithSpanNameFormatter] to include the url path in the span
 // names generated by an [otelhttp.Transport].

diff --git a/go.mod b/go.mod
@@ -178,7 +178,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0
-	go.opentelemetry.io/otel/metric v1.24.0
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.24.0
 	go.opentelemetry.io/otel/trace v1.24.0
 	go.opentelemetry.io/proto/otlp v1.1.0

diff --git a/lib/observability/tracing/tracing.go b/lib/observability/tracing/tracing.go
@@ -30,7 +30,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
-	metricnoop "go.opentelemetry.io/otel/metric/noop"
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
@@ -216,12 +215,6 @@ func NewTraceProvider(ctx context.Context, cfg Config) (*Provider, error) {
 	// set global propagator, the default is no-op.
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
 
-	// Set the global metric provider to a no-op so that any metrics created from otelgrpc interceptors
-	// are disabled to prevent memory leaks.
-	// See https://github.com/gravitational/teleport/issues/30759
-	// See https://github.com/open-telemetry/opentelemetry-go-contrib/issues/4226
-	otel.SetMeterProvider(metricnoop.MeterProvider{})
-
 	// override the global logging handled with one that uses the
 	// configured logger instead
 	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {