From 03d6be5e1b9ec82c5a9308b50d79d9b4593feb3e Mon Sep 17 00:00:00 2001
From: Chris Bandy <chris.bandy@crunchydata.com>
Date: Wed, 27 Nov 2024 22:54:37 -0600
Subject: [PATCH 1/3] Flush OpenTelemetry data with a deadline

Issue: PGO-1954
---
 cmd/postgres-operator/main.go           | 80 ++++++++++++++++++-------
 cmd/postgres-operator/main_test.go      | 17 +++---
 cmd/postgres-operator/open_telemetry.go | 12 ++--
 internal/controller/runtime/runtime.go  |  6 --
 4 files changed, 76 insertions(+), 39 deletions(-)

diff --git a/cmd/postgres-operator/main.go b/cmd/postgres-operator/main.go
index 143e420597..acdf3c9776 100644
--- a/cmd/postgres-operator/main.go
+++ b/cmd/postgres-operator/main.go
@@ -9,8 +9,10 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 	"unicode"
 
@@ -58,8 +60,8 @@ func initLogging() {
 
 //+kubebuilder:rbac:groups="coordination.k8s.io",resources="leases",verbs={get,create,update,watch}
 
-func initManager() (runtime.Options, error) {
-	log := logging.FromContext(context.Background())
+func initManager(ctx context.Context) (runtime.Options, error) {
+	log := logging.FromContext(ctx)
 
 	options := runtime.Options{}
 	options.Cache.SyncPeriod = initialize.Pointer(time.Hour)
@@ -120,33 +122,51 @@ func initManager() (runtime.Options, error) {
 }
 
 func main() {
-	// This context is canceled by SIGINT, SIGTERM, or by calling shutdown.
-	ctx, shutdown := context.WithCancel(runtime.SignalHandler())
-
-	otelFlush, err := initOpenTelemetry()
-	assertNoError(err)
-	defer otelFlush()
+	running, stopRunning := context.WithCancel(context.Background())
+	defer stopRunning()
 
 	initLogging()
-
-	log := logging.FromContext(ctx)
+	log := logging.FromContext(running)
 	log.V(1).Info("debug flag set to true")
 
+	// Start a goroutine that waits for SIGINT or SIGTERM.
+	{
+		signals := []os.Signal{os.Interrupt, syscall.SIGTERM}
+		receive := make(chan os.Signal, len(signals))
+		signal.Notify(receive, signals...)
+		go func() {
+			// Wait for a signal then immediately restore the default signal handlers.
+			// After this, a SIGHUP, SIGINT, or SIGTERM causes the program to exit.
+			// - https://pkg.go.dev/os/signal#hdr-Default_behavior_of_signals_in_Go_programs
+			s := <-receive
+			signal.Stop(receive)
+
+			log.Info("received signal from OS", "signal", s.String())
+			stopRunning()
+		}()
+	}
+
 	features := feature.NewGate()
 	assertNoError(features.Set(os.Getenv("PGO_FEATURE_GATES")))
 
-	ctx = feature.NewContext(ctx, features)
+	running = feature.NewContext(running, features)
 	log.Info("feature gates",
 		// These are set by the user
-		"PGO_FEATURE_GATES", feature.ShowAssigned(ctx),
+		"PGO_FEATURE_GATES", feature.ShowAssigned(running),
 		// These are enabled, including features that are on by default
-		"enabled", feature.ShowEnabled(ctx))
+		"enabled", feature.ShowEnabled(running))
+
+	// Initialize OpenTelemetry and flush data when there is a panic.
+	otelFinish, err := initOpenTelemetry(running)
+	assertNoError(err)
+	defer otelFinish(running)
 
 	cfg, err := runtime.GetConfig()
 	assertNoError(err)
 
 	cfg.Wrap(otelTransportWrapper())
 
+	// TODO(controller-runtime): Set config.WarningHandler instead after v0.19.0.
 	// Configure client-go to suppress warnings when warning headers are encountered. This prevents
 	// warnings from being logged over and over again during reconciliation (e.g. this will suppress
 	// deprecation warnings when using an older version of a resource for backwards compatibility).
@@ -154,11 +174,11 @@ func main() {
 
 	k8s, err := kubernetes.NewDiscoveryRunner(cfg)
 	assertNoError(err)
-	assertNoError(k8s.Read(ctx))
+	assertNoError(k8s.Read(running))
 
-	log.Info("Connected to Kubernetes", "api", k8s.Version().String(), "openshift", k8s.IsOpenShift())
+	log.Info("connected to Kubernetes", "api", k8s.Version().String(), "openshift", k8s.IsOpenShift())
 
-	options, err := initManager()
+	options, err := initManager(running)
 	assertNoError(err)
 
 	// Add to the Context that Manager passes to Reconciler.Start, Runnable.Start,
@@ -174,7 +194,7 @@ func main() {
 	assertNoError(err)
 	assertNoError(mgr.Add(k8s))
 
-	registrar, err := registration.NewRunner(os.Getenv("RSA_KEY"), os.Getenv("TOKEN_PATH"), shutdown)
+	registrar, err := registration.NewRunner(os.Getenv("RSA_KEY"), os.Getenv("TOKEN_PATH"), stopRunning)
 	assertNoError(err)
 	assertNoError(mgr.Add(registrar))
 	token, _ := registrar.CheckToken()
@@ -212,10 +232,30 @@ func main() {
 	assertNoError(mgr.AddHealthzCheck("health", healthz.Ping))
 	assertNoError(mgr.AddReadyzCheck("check", healthz.Ping))
 
-	log.Info("starting controller runtime manager and will wait for signal to exit")
+	// Start the manager and wait for its context to be canceled.
+	stopped := make(chan error, 1)
+	go func() { stopped <- mgr.Start(running) }()
+	<-running.Done()
+
+	// Set a deadline for graceful termination.
+	log.Info("shutting down")
+	stopping, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	// Wait for the manager to return or the deadline to pass.
+	select {
+	case err = <-stopped:
+	case <-stopping.Done():
+		err = stopping.Err()
+	}
 
-	assertNoError(mgr.Start(ctx))
-	log.Info("signal received, exiting")
+	// Flush any telemetry with the remaining time we have.
+	otelFinish(stopping)
+	if err != nil {
+		log.Error(err, "shutdown failed")
+	} else {
+		log.Info("shutdown complete")
+	}
 }
 
 // addControllersToManager adds all PostgreSQL Operator controllers to the provided controller
diff --git a/cmd/postgres-operator/main_test.go b/cmd/postgres-operator/main_test.go
index f369ce6bd3..a36cd21a13 100644
--- a/cmd/postgres-operator/main_test.go
+++ b/cmd/postgres-operator/main_test.go
@@ -5,6 +5,7 @@
 package main
 
 import (
+	"context"
 	"reflect"
 	"testing"
 	"time"
@@ -14,8 +15,10 @@ import (
 )
 
 func TestInitManager(t *testing.T) {
+	ctx := context.Background()
+
 	t.Run("Defaults", func(t *testing.T) {
-		options, err := initManager()
+		options, err := initManager(ctx)
 		assert.NilError(t, err)
 
 		if assert.Check(t, options.Cache.SyncPeriod != nil) {
@@ -48,7 +51,7 @@ func TestInitManager(t *testing.T) {
 		t.Run("Invalid", func(t *testing.T) {
 			t.Setenv("PGO_CONTROLLER_LEASE_NAME", "INVALID_NAME")
 
-			options, err := initManager()
+			options, err := initManager(ctx)
 			assert.ErrorContains(t, err, "PGO_CONTROLLER_LEASE_NAME")
 			assert.ErrorContains(t, err, "invalid")
 
@@ -59,7 +62,7 @@ func TestInitManager(t *testing.T) {
 		t.Run("Valid", func(t *testing.T) {
 			t.Setenv("PGO_CONTROLLER_LEASE_NAME", "valid-name")
 
-			options, err := initManager()
+			options, err := initManager(ctx)
 			assert.NilError(t, err)
 			assert.Assert(t, options.LeaderElection == true)
 			assert.Equal(t, options.LeaderElectionNamespace, "test-namespace")
@@ -70,7 +73,7 @@ func TestInitManager(t *testing.T) {
 	t.Run("PGO_TARGET_NAMESPACE", func(t *testing.T) {
 		t.Setenv("PGO_TARGET_NAMESPACE", "some-such")
 
-		options, err := initManager()
+		options, err := initManager(ctx)
 		assert.NilError(t, err)
 		assert.Assert(t, cmp.Len(options.Cache.DefaultNamespaces, 1),
 			"expected only one configured namespace")
@@ -81,7 +84,7 @@ func TestInitManager(t *testing.T) {
 	t.Run("PGO_TARGET_NAMESPACES", func(t *testing.T) {
 		t.Setenv("PGO_TARGET_NAMESPACES", "some-such,another-one")
 
-		options, err := initManager()
+		options, err := initManager(ctx)
 		assert.NilError(t, err)
 		assert.Assert(t, cmp.Len(options.Cache.DefaultNamespaces, 2),
 			"expect two configured namespaces")
@@ -95,7 +98,7 @@ func TestInitManager(t *testing.T) {
 			for _, v := range []string{"-3", "0", "3.14"} {
 				t.Setenv("PGO_WORKERS", v)
 
-				options, err := initManager()
+				options, err := initManager(ctx)
 				assert.NilError(t, err)
 				assert.DeepEqual(t, options.Controller.GroupKindConcurrency,
 					map[string]int{
@@ -107,7 +110,7 @@ func TestInitManager(t *testing.T) {
 		t.Run("Valid", func(t *testing.T) {
 			t.Setenv("PGO_WORKERS", "19")
 
-			options, err := initManager()
+			options, err := initManager(ctx)
 			assert.NilError(t, err)
 			assert.DeepEqual(t, options.Controller.GroupKindConcurrency,
 				map[string]int{
diff --git a/cmd/postgres-operator/open_telemetry.go b/cmd/postgres-operator/open_telemetry.go
index 2c9eedc135..2cd26a6608 100644
--- a/cmd/postgres-operator/open_telemetry.go
+++ b/cmd/postgres-operator/open_telemetry.go
@@ -19,7 +19,7 @@ import (
 	"go.opentelemetry.io/otel/sdk/trace"
 )
 
-func initOpenTelemetry() (func(), error) {
+func initOpenTelemetry(ctx context.Context) (func(context.Context), error) {
 	// At the time of this writing, the SDK (go.opentelemetry.io/otel@v1.2.0)
 	// does not automatically initialize any exporter. We import the OTLP and
 	// stdout exporters and configure them below. Much of the OTLP exporter can
@@ -49,8 +49,8 @@ func initOpenTelemetry() (func(), error) {
 		}
 
 		provider := trace.NewTracerProvider(trace.WithBatcher(exporter))
-		flush := func() {
-			_ = provider.Shutdown(context.TODO())
+		flush := func(ctx context.Context) {
+			_ = provider.Shutdown(ctx)
 			if closer != nil {
 				_ = closer.Close()
 			}
@@ -67,8 +67,8 @@ func initOpenTelemetry() (func(), error) {
 		}
 
 		provider := trace.NewTracerProvider(trace.WithBatcher(exporter))
-		flush := func() {
-			_ = provider.Shutdown(context.TODO())
+		flush := func(ctx context.Context) {
+			_ = provider.Shutdown(ctx)
 		}
 
 		otel.SetTracerProvider(provider)
@@ -78,7 +78,7 @@ func initOpenTelemetry() (func(), error) {
 	// $OTEL_TRACES_EXPORTER is unset or unknown, so no TracerProvider has been assigned.
 	// The default at this time is a single "no-op" tracer.
 
-	return func() {}, nil
+	return func(context.Context) {}, nil
 }
 
 // otelTransportWrapper creates a function that wraps the provided net/http.RoundTripper
diff --git a/internal/controller/runtime/runtime.go b/internal/controller/runtime/runtime.go
index 34bfeabf61..51fc37bf0d 100644
--- a/internal/controller/runtime/runtime.go
+++ b/internal/controller/runtime/runtime.go
@@ -5,8 +5,6 @@
 package runtime
 
 import (
-	"context"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -14,7 +12,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	"github.com/crunchydata/postgres-operator/internal/logging"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
@@ -71,6 +68,3 @@ func NewManager(config *rest.Config, options manager.Options) (manager.Manager,
 
 // SetLogger assigns the default Logger used by [sigs.k8s.io/controller-runtime].
 func SetLogger(logger logging.Logger) { log.SetLogger(logger) }
-
-// SignalHandler returns a Context that is canceled on SIGINT or SIGTERM.
-func SignalHandler() context.Context { return signals.SetupSignalHandler() }

From 7101a90a09c0c2540327420d9e2ef811914b6023 Mon Sep 17 00:00:00 2001
From: Chris Bandy <chris.bandy@crunchydata.com>
Date: Wed, 27 Nov 2024 23:55:19 -0600
Subject: [PATCH 2/3] Update OpenTelemetry and use an internal package

The "autoexport" and "autoprop" packages import and configure standard
exporters according to the current Specification, 1.39.0. The correct
correlation of logs to trace context uses underscores now.

The "google.golang.org/grpc" module raised its required Go version to
1.22.7 this release but has since reverted that change.

The "spancheck" linter reminds us to call "Span.End" after calling "tracing.Start".

Issue: PGO-1954
See: https://opentelemetry.io/docs/specs/otel
See: https://opentelemetry.io/docs/specs/otel/compatibility/logging_trace_context
See: https://www.github.com/open-telemetry/opentelemetry-go/issues/5969
---
 .golangci.yaml                                |  10 ++
 cmd/postgres-operator/main.go                 |  15 +-
 cmd/postgres-operator/open_telemetry.go       | 118 ++++++++-------
 cmd/postgres-operator/version.go              |  26 ++++
 go.mod                                        |  66 ++++++---
 go.sum                                        | 137 +++++++++++-------
 .../crunchybridgecluster_controller.go        |   6 +-
 .../pgupgrade/pgupgrade_controller.go         |   6 +-
 .../postgrescluster/cluster_test.go           |   3 -
 .../controller/postgrescluster/controller.go  |   5 +-
 .../postgrescluster/controller_test.go        |   2 -
 .../controller/postgrescluster/instance.go    |  23 ++-
 .../postgrescluster/instance_rollout_test.go  |  32 ++--
 .../postgrescluster/instance_test.go          |   2 -
 .../postgrescluster/pgbackrest_test.go        |   8 -
 .../standalone_pgadmin/controller.go          |   3 +
 internal/kubernetes/discovery.go              |   1 +
 internal/logging/logr.go                      |   8 +-
 internal/logging/logr_test.go                 |  16 +-
 internal/naming/dns.go                        |   4 +-
 internal/naming/telemetry.go                  |   9 --
 internal/tracing/tracing.go                   |  65 +++++++++
 internal/tracing/tracing_test.go              | 110 ++++++++++++++
 23 files changed, 468 insertions(+), 207 deletions(-)
 create mode 100644 cmd/postgres-operator/version.go
 delete mode 100644 internal/naming/telemetry.go
 create mode 100644 internal/tracing/tracing.go
 create mode 100644 internal/tracing/tracing_test.go

diff --git a/.golangci.yaml b/.golangci.yaml
index d886a4fb1e..e2dd0c9fb0 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -22,7 +22,13 @@ linters-settings:
   depguard:
     rules:
       everything:
+        list-mode: lax
+        allow:
+          - go.opentelemetry.io/otel/semconv/v1.27.0
         deny:
+          - pkg: go.opentelemetry.io/otel/semconv
+            desc: Use "go.opentelemetry.io/otel/semconv/v1.27.0" instead.
+
           - pkg: io/ioutil
             desc: >
               Use the "io" and "os" packages instead.
@@ -93,6 +99,10 @@ linters-settings:
         alias: apierrors
     no-unaliased: true
 
+  spancheck:
+    extra-start-span-signatures:
+      - 'github.com/crunchydata/postgres-operator/internal/tracing.Start:opentelemetry'
+
 issues:
   exclude-generated: strict
   exclude-rules:
diff --git a/cmd/postgres-operator/main.go b/cmd/postgres-operator/main.go
index acdf3c9776..908a04bb74 100644
--- a/cmd/postgres-operator/main.go
+++ b/cmd/postgres-operator/main.go
@@ -6,6 +6,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -16,7 +17,6 @@ import (
 	"time"
 	"unicode"
 
-	"go.opentelemetry.io/otel"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -33,12 +33,11 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/logging"
 	"github.com/crunchydata/postgres-operator/internal/naming"
 	"github.com/crunchydata/postgres-operator/internal/registration"
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 	"github.com/crunchydata/postgres-operator/internal/upgradecheck"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
-var versionString string
-
 // assertNoError panics when err is not nil.
 func assertNoError(err error) {
 	if err != nil {
@@ -125,6 +124,7 @@ func main() {
 	running, stopRunning := context.WithCancel(context.Background())
 	defer stopRunning()
 
+	initVersion()
 	initLogging()
 	log := logging.FromContext(running)
 	log.V(1).Info("debug flag set to true")
@@ -159,11 +159,14 @@ func main() {
 	// Initialize OpenTelemetry and flush data when there is a panic.
 	otelFinish, err := initOpenTelemetry(running)
 	assertNoError(err)
-	defer otelFinish(running)
+	defer func(ctx context.Context) { _ = otelFinish(ctx) }(running)
+
+	tracing.SetDefaultTracer(tracing.New("github.com/CrunchyData/postgres-operator"))
 
 	cfg, err := runtime.GetConfig()
 	assertNoError(err)
 
+	cfg.UserAgent = userAgent
 	cfg.Wrap(otelTransportWrapper())
 
 	// TODO(controller-runtime): Set config.WarningHandler instead after v0.19.0.
@@ -250,8 +253,7 @@ func main() {
 	}
 
 	// Flush any telemetry with the remaining time we have.
-	otelFinish(stopping)
-	if err != nil {
+	if err = errors.Join(err, otelFinish(stopping)); err != nil {
 		log.Error(err, "shutdown failed")
 	} else {
 		log.Info("shutdown complete")
@@ -266,7 +268,6 @@ func addControllersToManager(mgr runtime.Manager, log logging.Logger, reg regist
 		Owner:        postgrescluster.ControllerName,
 		Recorder:     mgr.GetEventRecorderFor(postgrescluster.ControllerName),
 		Registration: reg,
-		Tracer:       otel.Tracer(postgrescluster.ControllerName),
 	}
 
 	if err := pgReconciler.SetupWithManager(mgr); err != nil {
diff --git a/cmd/postgres-operator/open_telemetry.go b/cmd/postgres-operator/open_telemetry.go
index 2cd26a6608..02b12b19fa 100644
--- a/cmd/postgres-operator/open_telemetry.go
+++ b/cmd/postgres-operator/open_telemetry.go
@@ -6,79 +6,91 @@ package main
 
 import (
 	"context"
-	"fmt"
-	"io"
+	"errors"
 	"net/http"
 	"os"
 
+	"go.opentelemetry.io/contrib/exporters/autoexport"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/contrib/propagators/autoprop"
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
-	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
-	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	"go.opentelemetry.io/otel/sdk/resource"
 	"go.opentelemetry.io/otel/sdk/trace"
-)
+	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
 
-func initOpenTelemetry(ctx context.Context) (func(context.Context), error) {
-	// At the time of this writing, the SDK (go.opentelemetry.io/otel@v1.2.0)
-	// does not automatically initialize any exporter. We import the OTLP and
-	// stdout exporters and configure them below. Much of the OTLP exporter can
-	// be configured through environment variables.
-	//
-	// - https://github.com/open-telemetry/opentelemetry-go/issues/2310
-	// - https://github.com/open-telemetry/opentelemetry-specification/blob/v1.8.0/specification/sdk-environment-variables.md
+	"github.com/crunchydata/postgres-operator/internal/logging"
+)
 
-	switch os.Getenv("OTEL_TRACES_EXPORTER") {
-	case "json":
-		var closer io.Closer
-		filename := os.Getenv("OTEL_JSON_FILE")
-		options := []stdouttrace.Option{}
+func initOpenTelemetry(ctx context.Context) (func(context.Context) error, error) {
+	var started []interface{ Shutdown(context.Context) error }
 
-		if filename != "" {
-			file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
-			if err != nil {
-				return nil, fmt.Errorf("unable to open exporter file: %w", err)
-			}
-			closer = file
-			options = append(options, stdouttrace.WithWriter(file))
+	// shutdown returns the results of calling all the Shutdown methods in started.
+	var shutdown = func(ctx context.Context) error {
+		var err error
+		for _, s := range started {
+			err = errors.Join(err, s.Shutdown(ctx))
 		}
+		started = nil
+		return err
+	}
 
-		exporter, err := stdouttrace.New(options...)
-		if err != nil {
-			return nil, fmt.Errorf("unable to initialize stdout exporter: %w", err)
-		}
+	// The default for OTEL_PROPAGATORS is "tracecontext,baggage".
+	otel.SetTextMapPropagator(autoprop.NewTextMapPropagator())
 
-		provider := trace.NewTracerProvider(trace.WithBatcher(exporter))
-		flush := func(ctx context.Context) {
-			_ = provider.Shutdown(ctx)
-			if closer != nil {
-				_ = closer.Close()
-			}
-		}
+	// Skip any remaining setup when OTEL_SDK_DISABLED is exactly "true".
+	// - https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables
+	if os.Getenv("OTEL_SDK_DISABLED") == "true" {
+		return shutdown, nil
+	}
 
-		otel.SetTracerProvider(provider)
-		return flush, nil
+	log := logging.FromContext(ctx).WithName("open-telemetry")
+	otel.SetLogger(log)
+	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
+		// TODO(events): Emit this as an event instead.
+		log.V(1).Info(semconv.ExceptionEventName,
+			string(semconv.ExceptionMessageKey), err)
+	}))
 
-	case "otlp":
-		client := otlptracehttp.NewClient()
-		exporter, err := otlptrace.New(context.TODO(), client)
-		if err != nil {
-			return nil, fmt.Errorf("unable to initialize OTLP exporter: %w", err)
-		}
+	// Build a resource from the OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables.
+	// - https://opentelemetry.io/docs/languages/go/resources
+	self, _ := resource.Merge(resource.NewSchemaless(
+		semconv.ServiceVersion(versionString),
+	), resource.Default())
 
-		provider := trace.NewTracerProvider(trace.WithBatcher(exporter))
-		flush := func(ctx context.Context) {
-			_ = provider.Shutdown(ctx)
+	// Provide defaults for some other detectable attributes.
+	if r, err := resource.New(ctx,
+		resource.WithProcessRuntimeName(),
+		resource.WithProcessRuntimeVersion(),
+		resource.WithProcessRuntimeDescription(),
+	); err == nil {
+		self, _ = resource.Merge(r, self)
+	}
+	if r, err := resource.New(ctx,
+		resource.WithHost(),
+		resource.WithOS(),
+	); err == nil {
+		self, _ = resource.Merge(r, self)
+	}
+
+	// The default for OTEL_TRACES_EXPORTER is "otlp" but we prefer "none".
+	// Only assign an exporter when the environment variable is set.
+	if os.Getenv("OTEL_TRACES_EXPORTER") != "" {
+		exporter, err := autoexport.NewSpanExporter(ctx)
+		if err != nil {
+			return nil, errors.Join(err, shutdown(ctx))
 		}
 
+		// The defaults for this batch processor come from the OTEL_BSP_* environment variables.
+		// - https://pkg.go.dev/go.opentelemetry.io/otel/sdk/internal/env
+		provider := trace.NewTracerProvider(
+			trace.WithBatcher(exporter),
+			trace.WithResource(self),
+		)
+		started = append(started, provider)
 		otel.SetTracerProvider(provider)
-		return flush, nil
 	}
 
-	// $OTEL_TRACES_EXPORTER is unset or unknown, so no TracerProvider has been assigned.
-	// The default at this time is a single "no-op" tracer.
-
-	return func(context.Context) {}, nil
+	return shutdown, nil
 }
 
 // otelTransportWrapper creates a function that wraps the provided net/http.RoundTripper
diff --git a/cmd/postgres-operator/version.go b/cmd/postgres-operator/version.go
new file mode 100644
index 0000000000..0b04ce95e8
--- /dev/null
+++ b/cmd/postgres-operator/version.go
@@ -0,0 +1,26 @@
+// Copyright 2017 - 2024 Crunchy Data Solutions, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+var userAgent string
+var versionString string
+
+func initVersion() {
+	command := "unknown"
+	if len(os.Args) > 0 && len(os.Args[0]) > 0 {
+		command = filepath.Base(os.Args[0])
+	}
+	if len(versionString) > 0 {
+		command += "/" + versionString
+	}
+	userAgent = fmt.Sprintf("%s (%s/%s)", command, runtime.GOOS, runtime.GOARCH)
+}
diff --git a/go.mod b/go.mod
index df4430df70..26856e4456 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/crunchydata/postgres-operator
 
-go 1.22.0
+go 1.22.7
 
 require (
 	github.com/go-logr/logr v1.4.2
@@ -14,14 +14,13 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/xdg-go/stringprep v1.0.2
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
-	go.opentelemetry.io/otel v1.28.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0
-	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0
-	go.opentelemetry.io/otel/sdk v1.28.0
-	go.opentelemetry.io/otel/trace v1.28.0
-	golang.org/x/crypto v0.27.0
+	go.opentelemetry.io/contrib/exporters/autoexport v0.57.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0
+	go.opentelemetry.io/contrib/propagators/autoprop v0.57.0
+	go.opentelemetry.io/otel v1.32.0
+	go.opentelemetry.io/otel/sdk v1.32.0
+	go.opentelemetry.io/otel/trace v1.32.0
+	golang.org/x/crypto v0.28.0
 	golang.org/x/tools v0.22.0
 	gotest.tools/v3 v3.1.0
 	k8s.io/api v0.31.0
@@ -54,38 +53,59 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/moby/spdystream v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/client_golang v1.20.5 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/common v0.60.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect
+	go.opentelemetry.io/contrib/propagators/aws v1.32.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.32.0 // indirect
+	go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 // indirect
+	go.opentelemetry.io/contrib/propagators/ot v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.54.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 // indirect
+	go.opentelemetry.io/otel/log v0.8.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/sdk/log v0.8.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect
 	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/net v0.29.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/term v0.24.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/oauth2 v0.23.0 // indirect
+	golang.org/x/sync v0.9.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
-	google.golang.org/grpc v1.66.2 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
+	google.golang.org/grpc v1.68.0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
diff --git a/go.sum b/go.sum
index 25698530d8..86b776257e 100644
--- a/go.sum
+++ b/go.sum
@@ -49,7 +49,6 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -62,8 +61,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 h1:ad0vkEBuk23VJzZR9nkLVG0YAoN9coASF1GusYX6AlU=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0/go.mod h1:igFoXX2ELCW06bol23DWPB5BEWfZISOzSP5K2sbLea0=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -72,12 +71,16 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw=
 github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8=
@@ -102,16 +105,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
-github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
-github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc=
+github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -128,25 +131,58 @@ github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyh
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
-go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I=
-go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
-go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0 h1:OiYdrCq1Ctwnovp6EofSPwlp5aGy4LgKNbkg7PtEUw8=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0/go.mod h1:DUFCmFkXr0VtAHl5Zq2JRx24G6ze5CAq8YfdD36RdX8=
-go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
-go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
-go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U=
-go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
-go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
-go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0=
-go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
-go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
+go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 h1:UW0+QyeyBVhn+COBec3nGhfnFe5lwB0ic1JBVjzhk0w=
+go.opentelemetry.io/contrib/bridges/prometheus v0.57.0/go.mod h1:ppciCHRLsyCio54qbzQv0E4Jyth/fLWDTJYfvWpcSVk=
+go.opentelemetry.io/contrib/exporters/autoexport v0.57.0 h1:jmTVJ86dP60C01K3slFQa2NQ/Aoi7zA+wy7vMOKD9H4=
+go.opentelemetry.io/contrib/exporters/autoexport v0.57.0/go.mod h1:EJBheUMttD/lABFyLXhce47Wr6DPWYReCzaZiXadH7g=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
+go.opentelemetry.io/contrib/propagators/autoprop v0.57.0 h1:bNPJOdT5154XxzeFmrh8R+PXnV4t3TZEczy8gHEpcpg=
+go.opentelemetry.io/contrib/propagators/autoprop v0.57.0/go.mod h1:Tb0j0mK+QatKdCxCKPN7CSzc7kx/q34/KaohJx/N96s=
+go.opentelemetry.io/contrib/propagators/aws v1.32.0 h1:NELzr8bW7a7aHVZj5gaep1PfkvoSCGx+1qNGZx/uhhU=
+go.opentelemetry.io/contrib/propagators/aws v1.32.0/go.mod h1:XKMrzHNka3eOA+nGEcNKYVL9s77TAhkwQEynYuaRFnQ=
+go.opentelemetry.io/contrib/propagators/b3 v1.32.0 h1:MazJBz2Zf6HTN/nK/s3Ru1qme+VhWU5hm83QxEP+dvw=
+go.opentelemetry.io/contrib/propagators/b3 v1.32.0/go.mod h1:B0s70QHYPrJwPOwD1o3V/R8vETNOG9N3qZf4LDYvA30=
+go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 h1:K/fOyTMD6GELKTIJBaJ9k3ppF2Njt8MeUGBOwfaWXXA=
+go.opentelemetry.io/contrib/propagators/jaeger v1.32.0/go.mod h1:ISE6hda//MTWvtngG7p4et3OCngsrTVfl7c6DjN17f8=
+go.opentelemetry.io/contrib/propagators/ot v1.32.0 h1:Poy02A4wOZubHyd2hpHPDgZW+rn6EIq0vCwTZJ6Lmu8=
+go.opentelemetry.io/contrib/propagators/ot v1.32.0/go.mod h1:cbhaURV+VR3NIMarzDYZU1RDEkXG1fNd1WMP1XCcGkY=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 h1:WzNab7hOOLzdDF/EoWCt4glhrbMPVMOO5JYTmpz36Ls=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0/go.mod h1:hKvJwTzJdp90Vh7p6q/9PAOd55dI6WA6sWj62a/JvSs=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 h1:S+LdBGiQXtJdowoJoQPEtI52syEP/JYBUpjO49EQhV8=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0/go.mod h1:5KXybFvPGds3QinJWQT7pmXf+TN5YIa7CNYObWRkj50=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 h1:j7ZSD+5yn+lo3sGV69nW04rRR0jhYnBwjuX3r0HvnK0=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0/go.mod h1:WXbYJTUaZXAbYd8lbgGuvih0yuCfOFC5RJoYnoLcGz8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 h1:t/Qur3vKSkUCcDVaSumWF2PKHt85pc7fRvFuoVT8qFU=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0/go.mod h1:Rl61tySSdcOJWoEgYZVtmnKdA0GeKrSqkHC1t+91CH8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 h1:IJFEoHiytixx8cMiVAO+GmHR6Frwu+u5Ur8njpFO6Ac=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0/go.mod h1:3rHrKNtLIoS0oZwkY2vxi+oJcwFRWdtUyRII+so45p8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 h1:9kV11HXBHZAvuPUZxmMWrH8hZn/6UnHX4K0mu36vNsU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0/go.mod h1:JyA0FHXe22E1NeNiHmVp7kFHglnexDQ7uRWDiiJ1hKQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9O88joYEaI47CnQkxO1XZdpoTF9fEnW2duIddhw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI=
+go.opentelemetry.io/otel/exporters/prometheus v0.54.0 h1:rFwzp68QMgtzu9PgP3jm9XaMICI6TsofWWPcBDKwlsU=
+go.opentelemetry.io/otel/exporters/prometheus v0.54.0/go.mod h1:QyjcV9qDP6VeK5qPyKETvNjmaaEc7+gqjh4SS0ZYzDU=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0 h1:CHXNXwfKWfzS65yrlB2PVds1IBZcdsX8Vepy9of0iRU=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0/go.mod h1:zKU4zUgKiaRxrdovSS2amdM5gOc59slmo/zJwGX+YBg=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.32.0 h1:SZmDnHcgp3zwlPBS2JX2urGYe/jBKEIT6ZedHRUyCz8=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.32.0/go.mod h1:fdWW0HtZJ7+jNpTKUR0GpMEDP69nR8YBJQxNiVCE3jk=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 h1:cC2yDI3IQd0Udsux7Qmq8ToKAx1XCilTQECZ0KDZyTw=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0/go.mod h1:2PD5Ex6z8CFzDbTdOlwyNIUywRr1DN0ospafJM1wJ+s=
+go.opentelemetry.io/otel/log v0.8.0 h1:egZ8vV5atrUWUbnSsHn6vB8R21G2wrKqNiDt3iWertk=
+go.opentelemetry.io/otel/log v0.8.0/go.mod h1:M9qvDdUTRCopJcGRKg57+JSQ9LgLBrwwfC32epk5NX8=
+go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
+go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
+go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4=
+go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU=
+go.opentelemetry.io/otel/sdk/log v0.8.0 h1:zg7GUYXqxk1jnGF/dTdLPrK06xJdrXgqgFLnI4Crxvs=
+go.opentelemetry.io/otel/sdk/log v0.8.0/go.mod h1:50iXr0UVwQrYS45KbruFrEt4LvAdCaWWgIrsN3ZQggo=
+go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU=
+go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
 go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
 go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -158,8 +194,8 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
-golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM=
 golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -170,30 +206,29 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
-golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
-golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
-golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
+golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
-golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
-golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
+golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
-golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -209,16 +244,16 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 h1:QW9+G6Fir4VcRXVH8x3LilNAb6cxBGLa6+GM4hRwexE=
-google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go.mod h1:kdrSS/OiLkPrNUpzD4aHgCq2rVuC/YRxok32HXZ4vRE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
-google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo=
-google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
+google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 h1:M0KvPgPmDZHPlbRbaNU1APr28TvwvvdUPlSv7PUvy8g=
+google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
+google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0=
+google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
diff --git a/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go b/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go
index 0390417c9f..49a0cc5557 100644
--- a/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go
+++ b/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go
@@ -25,7 +25,9 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/bridge"
 	"github.com/crunchydata/postgres-operator/internal/controller/runtime"
 	"github.com/crunchydata/postgres-operator/internal/initialize"
+	"github.com/crunchydata/postgres-operator/internal/logging"
 	"github.com/crunchydata/postgres-operator/internal/naming"
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -100,7 +102,9 @@ func (r *CrunchyBridgeClusterReconciler) setControllerReference(
 // Reconcile does the work to move the current state of the world toward the
 // desired state described in a [v1beta1.CrunchyBridgeCluster] identified by req.
 func (r *CrunchyBridgeClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	log := ctrl.LoggerFrom(ctx)
+	ctx, span := tracing.Start(ctx, "reconcile-crunchybridgecluster")
+	log := logging.FromContext(ctx)
+	defer span.End()
 
 	// Retrieve the crunchybridgecluster from the client cache, if it exists. A deferred
 	// function below will send any changes to its Status field.
diff --git a/internal/controller/pgupgrade/pgupgrade_controller.go b/internal/controller/pgupgrade/pgupgrade_controller.go
index 0717607d7e..c66494c6be 100644
--- a/internal/controller/pgupgrade/pgupgrade_controller.go
+++ b/internal/controller/pgupgrade/pgupgrade_controller.go
@@ -20,7 +20,9 @@ import (
 
 	"github.com/crunchydata/postgres-operator/internal/config"
 	"github.com/crunchydata/postgres-operator/internal/controller/runtime"
+	"github.com/crunchydata/postgres-operator/internal/logging"
 	"github.com/crunchydata/postgres-operator/internal/registration"
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -93,7 +95,9 @@ func (r *PGUpgradeReconciler) findUpgradesForPostgresCluster(
 // Reconcile does the work to move the current state of the world toward the
 // desired state described in a [v1beta1.PGUpgrade] identified by req.
 func (r *PGUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, err error) {
-	log := ctrl.LoggerFrom(ctx)
+	ctx, span := tracing.Start(ctx, "reconcile-pgupgrade")
+	log := logging.FromContext(ctx)
+	defer span.End()
 
 	// Retrieve the upgrade from the client cache, if it exists. A deferred
 	// function below will send any changes to its Status field.
diff --git a/internal/controller/postgrescluster/cluster_test.go b/internal/controller/postgrescluster/cluster_test.go
index 491add9f34..3ef98c58cf 100644
--- a/internal/controller/postgrescluster/cluster_test.go
+++ b/internal/controller/postgrescluster/cluster_test.go
@@ -8,7 +8,6 @@ import (
 	"context"
 	"testing"
 
-	"go.opentelemetry.io/otel"
 	"gotest.tools/v3/assert"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -83,7 +82,6 @@ func TestCustomLabels(t *testing.T) {
 		Client:   cc,
 		Owner:    client.FieldOwner(t.Name()),
 		Recorder: new(record.FakeRecorder),
-		Tracer:   otel.Tracer(t.Name()),
 	}
 
 	ns := setupNamespace(t, cc)
@@ -322,7 +320,6 @@ func TestCustomAnnotations(t *testing.T) {
 		Client:   cc,
 		Owner:    client.FieldOwner(t.Name()),
 		Recorder: new(record.FakeRecorder),
-		Tracer:   otel.Tracer(t.Name()),
 	}
 
 	ns := setupNamespace(t, cc)
diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go
index 394c87a750..512738621c 100644
--- a/internal/controller/postgrescluster/controller.go
+++ b/internal/controller/postgrescluster/controller.go
@@ -11,7 +11,6 @@ import (
 	"io"
 	"time"
 
-	"go.opentelemetry.io/otel/trace"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -40,6 +39,7 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/pki"
 	"github.com/crunchydata/postgres-operator/internal/postgres"
 	"github.com/crunchydata/postgres-operator/internal/registration"
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -58,7 +58,6 @@ type Reconciler struct {
 	) error
 	Recorder     record.EventRecorder
 	Registration registration.Registration
-	Tracer       trace.Tracer
 }
 
 // +kubebuilder:rbac:groups="",resources="events",verbs={create,patch}
@@ -69,7 +68,7 @@ type Reconciler struct {
 func (r *Reconciler) Reconcile(
 	ctx context.Context, request reconcile.Request) (reconcile.Result, error,
 ) {
-	ctx, span := r.Tracer.Start(ctx, "Reconcile")
+	ctx, span := tracing.Start(ctx, "reconcile-postgrescluster")
 	log := logging.FromContext(ctx)
 	defer span.End()
 
diff --git a/internal/controller/postgrescluster/controller_test.go b/internal/controller/postgrescluster/controller_test.go
index b9e928ecce..6def47556e 100644
--- a/internal/controller/postgrescluster/controller_test.go
+++ b/internal/controller/postgrescluster/controller_test.go
@@ -15,7 +15,6 @@ import (
 	. "github.com/onsi/gomega/gstruct"
 	"github.com/pkg/errors" //nolint:depguard // This legacy test covers so much code, it logs the origin of unexpected errors.
 
-	"go.opentelemetry.io/otel"
 	"gotest.tools/v3/assert"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -138,7 +137,6 @@ var _ = Describe("PostgresCluster Reconciler", func() {
 		test.Reconciler.Owner = "asdf"
 		test.Reconciler.Recorder = test.Recorder
 		test.Reconciler.Registration = nil
-		test.Reconciler.Tracer = otel.Tracer("asdf")
 	})
 
 	AfterEach(func() {
diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go
index ff3810ae3c..4dfa37559c 100644
--- a/internal/controller/postgrescluster/instance.go
+++ b/internal/controller/postgrescluster/instance.go
@@ -13,8 +13,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/trace"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"
@@ -36,6 +34,7 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/pgbackrest"
 	"github.com/crunchydata/postgres-operator/internal/pki"
 	"github.com/crunchydata/postgres-operator/internal/postgres"
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -800,8 +799,7 @@ func (r *Reconciler) rolloutInstance(
 	// NOTE(cbandy): The StatefulSet controlling this Pod reflects this change
 	// in its Status and triggers another reconcile.
 	if primary && len(instances.forCluster) > 1 {
-		var span trace.Span
-		ctx, span = r.Tracer.Start(ctx, "patroni-change-primary")
+		ctx, span := tracing.Start(ctx, "patroni-change-primary")
 		defer span.End()
 
 		success, err := patroni.Executor(exec).ChangePrimaryAndWait(ctx, pod.Name, "")
@@ -824,7 +822,7 @@ func (r *Reconciler) rolloutInstance(
 		}
 
 		checkpoint := func(ctx context.Context) (time.Duration, error) {
-			ctx, span := r.Tracer.Start(ctx, "postgresql-checkpoint")
+			ctx, span := tracing.Start(ctx, "postgresql-checkpoint")
 			defer span.End()
 
 			start := time.Now()
@@ -894,7 +892,7 @@ func (r *Reconciler) rolloutInstances(
 	var numAvailable int
 	var numSpecified int
 
-	ctx, span := r.Tracer.Start(ctx, "rollout-instances")
+	ctx, span := tracing.Start(ctx, "rollout-instances")
 	defer span.End()
 
 	for _, set := range cluster.Spec.InstanceSets {
@@ -933,12 +931,10 @@ func (r *Reconciler) rolloutInstances(
 		sort.Sort(byPriority(consider))
 	}
 
-	span.SetAttributes(
-		attribute.Int("instances", len(instances.forCluster)),
-		attribute.Int("specified", numSpecified),
-		attribute.Int("available", numAvailable),
-		attribute.Int("considering", len(consider)),
-	)
+	tracing.Int(span, "instances", len(instances.forCluster))
+	tracing.Int(span, "specified", numSpecified)
+	tracing.Int(span, "available", numAvailable)
+	tracing.Int(span, "considering", len(consider))
 
 	// Redeploy instances up to the allowed maximum while "rolling over" any
 	// unavailable instances.
@@ -1085,8 +1081,7 @@ func (r *Reconciler) scaleUpInstances(
 	// While there are fewer instances than specified, generate another empty one
 	// and append it.
 	for len(instances) < int(*set.Replicas) {
-		var span trace.Span
-		ctx, span = r.Tracer.Start(ctx, "generateInstanceName")
+		_, span := tracing.Start(ctx, "generate-instance-name")
 		next := naming.GenerateInstance(cluster, set)
 		// if there are any available instance names (as determined by observing any PVCs for the
 		// instance set that are not currently associated with an instance, e.g. in the event the
diff --git a/internal/controller/postgrescluster/instance_rollout_test.go b/internal/controller/postgrescluster/instance_rollout_test.go
index e668907497..bede908615 100644
--- a/internal/controller/postgrescluster/instance_rollout_test.go
+++ b/internal/controller/postgrescluster/instance_rollout_test.go
@@ -10,7 +10,6 @@ import (
 	"strings"
 	"testing"
 
-	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/sdk/trace"
 	"go.opentelemetry.io/otel/sdk/trace/tracetest"
@@ -25,6 +24,7 @@ import (
 
 	"github.com/crunchydata/postgres-operator/internal/initialize"
 	"github.com/crunchydata/postgres-operator/internal/testing/cmp"
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -60,7 +60,6 @@ func TestReconcilerRolloutInstance(t *testing.T) {
 		key := client.ObjectKey{Namespace: "ns1", Name: "one-pod-bruh"}
 		reconciler := &Reconciler{}
 		reconciler.Client = fake.NewClientBuilder().WithObjects(instances[0].Pods[0]).Build()
-		reconciler.Tracer = otel.Tracer(t.Name())
 
 		execCalls := 0
 		reconciler.PodExec = func(
@@ -121,7 +120,6 @@ func TestReconcilerRolloutInstance(t *testing.T) {
 		t.Run("Success", func(t *testing.T) {
 			execCalls := 0
 			reconciler := &Reconciler{}
-			reconciler.Tracer = otel.Tracer(t.Name())
 			reconciler.PodExec = func(
 				ctx context.Context, namespace, pod, container string, _ io.Reader, stdout, _ io.Writer, command ...string,
 			) error {
@@ -149,7 +147,6 @@ func TestReconcilerRolloutInstance(t *testing.T) {
 
 		t.Run("Failure", func(t *testing.T) {
 			reconciler := &Reconciler{}
-			reconciler.Tracer = otel.Tracer(t.Name())
 			reconciler.PodExec = func(
 				ctx context.Context, _, _, _ string, _ io.Reader, _, _ io.Writer, _ ...string,
 			) error {
@@ -165,26 +162,25 @@ func TestReconcilerRolloutInstance(t *testing.T) {
 
 func TestReconcilerRolloutInstances(t *testing.T) {
 	ctx := context.Background()
-	reconciler := &Reconciler{Tracer: otel.Tracer(t.Name())}
+	reconciler := &Reconciler{}
 
 	accumulate := func(on *[]*Instance) func(context.Context, *Instance) error {
 		return func(_ context.Context, i *Instance) error { *on = append(*on, i); return nil }
 	}
 
-	logSpanAttributes := func(t testing.TB) {
+	logSpanAttributes := func(t testing.TB, ctx context.Context) context.Context {
 		recorder := tracetest.NewSpanRecorder()
 		provider := trace.NewTracerProvider(trace.WithSpanProcessor(recorder))
-
-		former := reconciler.Tracer
-		reconciler.Tracer = provider.Tracer(t.Name())
+		tracer := provider.Tracer(t.Name())
 
 		t.Cleanup(func() {
-			reconciler.Tracer = former
 			for _, span := range recorder.Ended() {
 				attr := attribute.NewSet(span.Attributes()...)
 				t.Log(span.Name(), attr.Encoded(attribute.DefaultEncoder()))
 			}
 		})
+
+		return tracing.NewContext(ctx, tracer)
 	}
 
 	// Nothing specified, nothing observed, nothing to do.
@@ -192,7 +188,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 		cluster := new(v1beta1.PostgresCluster)
 		observed := new(observedInstances)
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed,
 			func(context.Context, *Instance) error {
 				t.Fatal("expected no redeploys")
@@ -237,7 +233,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 		}
 		observed := &observedInstances{forCluster: instances}
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed,
 			func(context.Context, *Instance) error {
 				t.Fatal("expected no redeploys")
@@ -284,7 +280,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 
 		var redeploys []*Instance
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed, accumulate(&redeploys)))
 		assert.Equal(t, len(redeploys), 1)
 		assert.Equal(t, redeploys[0].Name, "one")
@@ -354,7 +350,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 
 		var redeploys []*Instance
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed, accumulate(&redeploys)))
 		assert.Equal(t, len(redeploys), 1)
 		assert.Equal(t, redeploys[0].Name, "one", `expected the "lowest" name`)
@@ -425,7 +421,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 
 		var redeploys []*Instance
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed, accumulate(&redeploys)))
 		assert.Equal(t, len(redeploys), 1)
 		assert.Equal(t, redeploys[0].Name, "not-primary")
@@ -495,7 +491,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 
 		var redeploys []*Instance
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed, accumulate(&redeploys)))
 		assert.Equal(t, len(redeploys), 1)
 		assert.Equal(t, redeploys[0].Name, "not-ready")
@@ -564,7 +560,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 		}
 		observed := &observedInstances{forCluster: instances}
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed,
 			func(context.Context, *Instance) error {
 				t.Fatal("expected no redeploys")
@@ -633,7 +629,7 @@ func TestReconcilerRolloutInstances(t *testing.T) {
 		}
 		observed := &observedInstances{forCluster: instances}
 
-		logSpanAttributes(t)
+		ctx := logSpanAttributes(t, ctx)
 		assert.NilError(t, reconciler.rolloutInstances(ctx, cluster, observed,
 			func(context.Context, *Instance) error {
 				t.Fatal("expected no redeploys")
diff --git a/internal/controller/postgrescluster/instance_test.go b/internal/controller/postgrescluster/instance_test.go
index f4eda5b056..064714872f 100644
--- a/internal/controller/postgrescluster/instance_test.go
+++ b/internal/controller/postgrescluster/instance_test.go
@@ -16,7 +16,6 @@ import (
 
 	"github.com/go-logr/logr/funcr"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	"go.opentelemetry.io/otel"
 	"gotest.tools/v3/assert"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -1339,7 +1338,6 @@ func TestDeleteInstance(t *testing.T) {
 		Client:   cc,
 		Owner:    client.FieldOwner(t.Name()),
 		Recorder: new(record.FakeRecorder),
-		Tracer:   otel.Tracer(t.Name()),
 	}
 
 	// Define, Create, and Reconcile a cluster to get an instance running in kube
diff --git a/internal/controller/postgrescluster/pgbackrest_test.go b/internal/controller/postgrescluster/pgbackrest_test.go
index c078f37d8a..b7855f1732 100644
--- a/internal/controller/postgrescluster/pgbackrest_test.go
+++ b/internal/controller/postgrescluster/pgbackrest_test.go
@@ -15,7 +15,6 @@ import (
 	"testing"
 	"time"
 
-	"go.opentelemetry.io/otel"
 	"gotest.tools/v3/assert"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -180,7 +179,6 @@ func TestReconcilePGBackRest(t *testing.T) {
 		r = &Reconciler{
 			Client:   mgr.GetClient(),
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
-			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
@@ -735,7 +733,6 @@ func TestReconcileStanzaCreate(t *testing.T) {
 		r = &Reconciler{
 			Client:   mgr.GetClient(),
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
-			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
@@ -1014,7 +1011,6 @@ func TestReconcileManualBackup(t *testing.T) {
 		r = &Reconciler{
 			Client:   mgr.GetClient(),
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
-			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
@@ -1724,7 +1720,6 @@ func TestReconcilePostgresClusterDataSource(t *testing.T) {
 		r = &Reconciler{
 			Client:   tClient,
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
-			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
@@ -2018,7 +2013,6 @@ func TestReconcileCloudBasedDataSource(t *testing.T) {
 		r = &Reconciler{
 			Client:   tClient,
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
-			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
@@ -3393,7 +3387,6 @@ func TestReconcileScheduledBackups(t *testing.T) {
 		r = &Reconciler{
 			Client:   mgr.GetClient(),
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
-			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
@@ -3730,7 +3723,6 @@ func TestBackupsEnabled(t *testing.T) {
 		r = &Reconciler{
 			Client:   mgr.GetClient(),
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
-			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
diff --git a/internal/controller/standalone_pgadmin/controller.go b/internal/controller/standalone_pgadmin/controller.go
index d16c33b797..481231684f 100644
--- a/internal/controller/standalone_pgadmin/controller.go
+++ b/internal/controller/standalone_pgadmin/controller.go
@@ -20,6 +20,7 @@ import (
 
 	"github.com/crunchydata/postgres-operator/internal/controller/runtime"
 	"github.com/crunchydata/postgres-operator/internal/logging"
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -83,7 +84,9 @@ func (r *PGAdminReconciler) SetupWithManager(mgr ctrl.Manager) error {
 func (r *PGAdminReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 
 	var err error
+	ctx, span := tracing.Start(ctx, "reconcile-pgadmin")
 	log := logging.FromContext(ctx)
+	defer span.End()
 
 	pgAdmin := &v1beta1.PGAdmin{}
 	if err := r.Get(ctx, req.NamespacedName, pgAdmin); err != nil {
diff --git a/internal/kubernetes/discovery.go b/internal/kubernetes/discovery.go
index ddc8d2cc3a..471e5360ea 100644
--- a/internal/kubernetes/discovery.go
+++ b/internal/kubernetes/discovery.go
@@ -26,6 +26,7 @@ type Version = version.Info
 type DiscoveryRunner struct {
 	// NOTE(tracing): The methods of [discovery.DiscoveryClient] do not take
 	// a Context so their API calls won't have a parent span.
+	// - https://issue.k8s.io/126379
 	Client interface {
 		ServerGroups() (*metav1.APIGroupList, error)
 		ServerResourcesForGroupVersion(string) (*metav1.APIResourceList, error)
diff --git a/internal/logging/logr.go b/internal/logging/logr.go
index 7d6f208744..4d82294dd6 100644
--- a/internal/logging/logr.go
+++ b/internal/logging/logr.go
@@ -37,10 +37,12 @@ func FromContext(ctx context.Context) Logger {
 	}
 
 	// Add trace context, if any, according to OpenTelemetry recommendations.
-	// Omit trace flags for now because they don't seem relevant.
-	// - https://github.com/open-telemetry/opentelemetry-specification/blob/v0.7.0/specification/logs/overview.md
+	// - https://github.com/open-telemetry/opentelemetry-specification/blob/v1.39.0/specification/compatibility/logging_trace_context.md
 	if sc := trace.SpanFromContext(ctx).SpanContext(); sc.IsValid() {
-		log = log.WithValues("spanid", sc.SpanID(), "traceid", sc.TraceID())
+		log = log.WithValues(
+			"span_id", sc.SpanID(),
+			"trace_id", sc.TraceID(),
+			"trace_flags", sc.TraceFlags())
 	}
 
 	return log
diff --git a/internal/logging/logr_test.go b/internal/logging/logr_test.go
index 1cbc818ad9..5b78c1dd7a 100644
--- a/internal/logging/logr_test.go
+++ b/internal/logging/logr_test.go
@@ -31,11 +31,11 @@ func TestFromContext(t *testing.T) {
 }
 
 func TestFromContextTraceContext(t *testing.T) {
-	var calls []map[string]interface{}
+	var calls []map[string]any
 
 	SetLogSink(&sink{
-		fnInfo: func(_ int, _ string, kv ...interface{}) {
-			m := make(map[string]interface{})
+		fnInfo: func(_ int, _ string, kv ...any) {
+			m := make(map[string]any)
 			for i := 0; i < len(kv); i += 2 {
 				m[kv[i].(string)] = kv[i+1]
 			}
@@ -47,23 +47,23 @@ func TestFromContextTraceContext(t *testing.T) {
 
 	// Nothing when there's no trace.
 	FromContext(ctx).Info("")
-	assert.Equal(t, calls[0]["spanid"], nil)
-	assert.Equal(t, calls[0]["traceid"], nil)
+	assert.Equal(t, calls[0]["span_id"], nil)
+	assert.Equal(t, calls[0]["trace_id"], nil)
 
 	ctx, span := trace.NewTracerProvider().Tracer("").Start(ctx, "test-span")
 	defer span.End()
 
 	// OpenTelemetry trace context when there is.
 	FromContext(ctx).Info("")
-	assert.Equal(t, calls[1]["spanid"], span.SpanContext().SpanID())
-	assert.Equal(t, calls[1]["traceid"], span.SpanContext().TraceID())
+	assert.Equal(t, calls[1]["span_id"], span.SpanContext().SpanID())
+	assert.Equal(t, calls[1]["trace_id"], span.SpanContext().TraceID())
 }
 
 func TestSetLogSink(t *testing.T) {
 	var calls []string
 
 	SetLogSink(&sink{
-		fnInfo: func(_ int, m string, _ ...interface{}) {
+		fnInfo: func(_ int, m string, _ ...any) {
 			calls = append(calls, m)
 		},
 	})
diff --git a/internal/naming/dns.go b/internal/naming/dns.go
index d3351a5d70..3d770bd2aa 100644
--- a/internal/naming/dns.go
+++ b/internal/naming/dns.go
@@ -11,6 +11,8 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+
+	"github.com/crunchydata/postgres-operator/internal/tracing"
 )
 
 // InstancePodDNSNames returns the possible DNS names for instance. The first
@@ -68,7 +70,7 @@ func ServiceDNSNames(ctx context.Context, service *corev1.Service) []string {
 
 // KubernetesClusterDomain looks up the Kubernetes cluster domain name.
 func KubernetesClusterDomain(ctx context.Context) string {
-	ctx, span := tracer.Start(ctx, "kubernetes-domain-lookup")
+	ctx, span := tracing.Start(ctx, "kubernetes-domain-lookup")
 	defer span.End()
 
 	// Lookup an existing Service to determine its fully qualified domain name.
diff --git a/internal/naming/telemetry.go b/internal/naming/telemetry.go
deleted file mode 100644
index 5825d6299f..0000000000
--- a/internal/naming/telemetry.go
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright 2021 - 2024 Crunchy Data Solutions, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package naming
-
-import "go.opentelemetry.io/otel"
-
-var tracer = otel.Tracer("github.com/crunchydata/postgres-operator/naming")
diff --git a/internal/tracing/tracing.go b/internal/tracing/tracing.go
new file mode 100644
index 0000000000..f7f722c8db
--- /dev/null
+++ b/internal/tracing/tracing.go
@@ -0,0 +1,65 @@
+// Copyright 2021 - 2024 Crunchy Data Solutions, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+// https://pkg.go.dev/go.opentelemetry.io/otel/trace
+type (
+	Span   = trace.Span
+	Tracer = trace.Tracer
+)
+
+var global = noop.NewTracerProvider().Tracer("")
+
+// SetDefaultTracer replaces the default Tracer with t. Before this is called,
+// the default Tracer is a no-op.
+func SetDefaultTracer(t Tracer) { global = t }
+
+type tracerKey struct{}
+
+// FromContext returns the Tracer stored by a prior call to [WithTracer] or [SetDefaultTracer].
+func FromContext(ctx context.Context) Tracer {
+	if t, ok := ctx.Value(tracerKey{}).(Tracer); ok {
+		return t
+	}
+	return global
+}
+
+// NewContext returns a copy of ctx containing t. Retrieve it using [FromContext].
+func NewContext(ctx context.Context, t Tracer) context.Context {
+	return context.WithValue(ctx, tracerKey{}, t)
+}
+
+// New returns a Tracer produced by [otel.GetTracerProvider].
+func New(name string, opts ...trace.TracerOption) Tracer {
+	opts = append([]trace.TracerOption{
+		trace.WithSchemaURL(semconv.SchemaURL),
+	}, opts...)
+
+	return otel.GetTracerProvider().Tracer(name, opts...)
+}
+
+// Start creates a Span and a Context containing it. It uses the Tracer returned by [FromContext].
+func Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, Span) {
+	return FromContext(ctx).Start(ctx, name, opts...)
+}
+
+// Bool sets the k attribute of s to v.
+func Bool(s Span, k string, v bool) { s.SetAttributes(attribute.Bool(k, v)) }
+
+// Int sets the k attribute of s to v.
+func Int(s Span, k string, v int) { s.SetAttributes(attribute.Int(k, v)) }
+
+// String sets the k attribute of s to v.
+func String(s Span, k, v string) { s.SetAttributes(attribute.String(k, v)) }
diff --git a/internal/tracing/tracing_test.go b/internal/tracing/tracing_test.go
new file mode 100644
index 0000000000..e9d519a71c
--- /dev/null
+++ b/internal/tracing/tracing_test.go
@@ -0,0 +1,110 @@
+// Copyright 2021 - 2024 Crunchy Data Solutions, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package tracing
+
+import (
+	"context"
+	"testing"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
+	"gotest.tools/v3/assert"
+)
+
+func TestDefaultTracer(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("no-op", func(t *testing.T) {
+		tracer := FromContext(ctx)
+		_, s1 := tracer.Start(ctx, "asdf")
+		defer s1.End()
+		assert.Assert(t, !s1.IsRecording())
+
+		_, s2 := Start(ctx, "doot")
+		defer s2.End()
+		assert.Assert(t, !s2.IsRecording())
+	})
+
+	t.Run("set", func(t *testing.T) {
+		prior := global
+		t.Cleanup(func() { SetDefaultTracer(prior) })
+
+		recorder := tracetest.NewSpanRecorder()
+		SetDefaultTracer(trace.NewTracerProvider(
+			trace.WithSpanProcessor(recorder),
+		).Tracer("myst"))
+
+		_, span := Start(ctx, "zork")
+		span.End()
+
+		spans := recorder.Ended()
+		assert.Equal(t, len(spans), 1)
+		assert.Equal(t, spans[0].InstrumentationScope().Name, "myst")
+		assert.Equal(t, spans[0].Name(), "zork")
+	})
+}
+
+func TestNew(t *testing.T) {
+	prior := otel.GetTracerProvider()
+	t.Cleanup(func() { otel.SetTracerProvider(prior) })
+
+	recorder := tracetest.NewSpanRecorder()
+	otel.SetTracerProvider(trace.NewTracerProvider(
+		trace.WithSpanProcessor(recorder),
+	))
+
+	_, span := New("onetwo").Start(context.Background(), "three")
+	span.End()
+
+	spans := recorder.Ended()
+	assert.Equal(t, len(spans), 1)
+	assert.Equal(t, spans[0].InstrumentationScope().Name, "onetwo")
+	assert.Equal(t, spans[0].InstrumentationScope().SchemaURL, semconv.SchemaURL)
+	assert.Equal(t, spans[0].Name(), "three")
+}
+
+func TestFromContext(t *testing.T) {
+	recorder := tracetest.NewSpanRecorder()
+
+	ctx := NewContext(context.Background(), trace.NewTracerProvider(
+		trace.WithSpanProcessor(recorder),
+	).Tracer("something"))
+
+	_, span := Start(ctx, "spanspan")
+	span.End()
+
+	spans := recorder.Ended()
+	assert.Equal(t, len(spans), 1)
+	assert.Equal(t, spans[0].InstrumentationScope().Name, "something")
+	assert.Equal(t, spans[0].Name(), "spanspan")
+}
+
+func TestAttributes(t *testing.T) {
+	recorder := tracetest.NewSpanRecorder()
+
+	ctx := NewContext(context.Background(), trace.NewTracerProvider(
+		trace.WithSpanProcessor(recorder),
+	).Tracer(""))
+
+	_, span := Start(ctx, "")
+	Bool(span, "aa", true)
+	Int(span, "abc", 99)
+	String(span, "xyz", "copy pasta")
+	span.End()
+
+	spans := recorder.Ended()
+	assert.Equal(t, len(spans), 1)
+	assert.Equal(t, len(spans[0].Attributes()), 3)
+
+	attrs := spans[0].Attributes()
+	assert.Equal(t, string(attrs[0].Key), "aa")
+	assert.Equal(t, string(attrs[1].Key), "abc")
+	assert.Equal(t, string(attrs[2].Key), "xyz")
+	assert.Equal(t, attrs[0].Value.AsInterface(), true)
+	assert.Equal(t, attrs[1].Value.AsInterface(), int64(99))
+	assert.Equal(t, attrs[2].Value.AsInterface(), "copy pasta")
+}

From 45c532fe1acf599742b1c6ea8660a15fc5ddab5e Mon Sep 17 00:00:00 2001
From: Chris Bandy <chris.bandy@crunchydata.com>
Date: Mon, 2 Dec 2024 15:32:39 -0600
Subject: [PATCH 3/3] Add functions for attaching errors to spans

The "spancheck" linter reminds us to call "Span.RecordError" when
returning an error. Two functions help with that: "tracing.Check" and
"tracing.Escape".
---
 .golangci.yaml                                |  3 +
 .../crunchybridgecluster_controller.go        | 16 ++--
 .../pgupgrade/pgupgrade_controller.go         |  1 +
 .../controller/postgrescluster/controller.go  | 21 ++---
 .../controller/postgrescluster/instance.go    | 38 ++++----
 .../standalone_pgadmin/controller.go          |  4 +-
 internal/naming/dns.go                        |  3 +-
 internal/tracing/errors.go                    | 34 +++++++
 internal/tracing/errors_test.go               | 94 +++++++++++++++++++
 9 files changed, 172 insertions(+), 42 deletions(-)
 create mode 100644 internal/tracing/errors.go
 create mode 100644 internal/tracing/errors_test.go

diff --git a/.golangci.yaml b/.golangci.yaml
index e2dd0c9fb0..59bf0ad535 100644
--- a/.golangci.yaml
+++ b/.golangci.yaml
@@ -100,8 +100,11 @@ linters-settings:
     no-unaliased: true
 
   spancheck:
+    checks: [end, record-error]
     extra-start-span-signatures:
       - 'github.com/crunchydata/postgres-operator/internal/tracing.Start:opentelemetry'
+    ignore-check-signatures:
+      - 'tracing.Escape'
 
 issues:
   exclude-generated: strict
diff --git a/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go b/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go
index 49a0cc5557..df283318c1 100644
--- a/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go
+++ b/internal/bridge/crunchybridgecluster/crunchybridgecluster_controller.go
@@ -133,7 +133,7 @@ func (r *CrunchyBridgeClusterReconciler) Reconcile(ctx context.Context, req ctrl
 		// NotFound cannot be fixed by requeuing so ignore it. During background
 		// deletion, we receive delete events from crunchybridgecluster's dependents after
 		// crunchybridgecluster is deleted.
-		return ctrl.Result{}, client.IgnoreNotFound(err)
+		return ctrl.Result{}, tracing.Escape(span, client.IgnoreNotFound(err))
 	}
 
 	// Get and validate connection secret for requests
@@ -152,12 +152,12 @@ func (r *CrunchyBridgeClusterReconciler) Reconcile(ctx context.Context, req ctrl
 	// is not being deleted.
 	if result, err := r.handleDelete(ctx, crunchybridgecluster, key); err != nil {
 		log.Error(err, "deleting")
-		return ctrl.Result{}, err
+		return ctrl.Result{}, tracing.Escape(span, err)
 	} else if result != nil {
 		if log := log.V(1); log.Enabled() {
 			log.Info("deleting", "result", fmt.Sprintf("%+v", *result))
 		}
-		return *result, err
+		return *result, tracing.Escape(span, err)
 	}
 
 	// Wonder if there's a better way to handle adding/checking/removing statuses
@@ -190,7 +190,7 @@ func (r *CrunchyBridgeClusterReconciler) Reconcile(ctx context.Context, req ctrl
 		// Check if a cluster with the same name already exists
 		controllerResult, err := r.handleDuplicateClusterName(ctx, key, team, crunchybridgecluster)
 		if err != nil || controllerResult != nil {
-			return *controllerResult, err
+			return *controllerResult, tracing.Escape(span, err)
 		}
 
 		// if we've gotten here then no cluster exists with that name and we're missing the ID, ergo, create cluster
@@ -204,26 +204,26 @@ func (r *CrunchyBridgeClusterReconciler) Reconcile(ctx context.Context, req ctrl
 	// Get Cluster
 	err = r.handleGetCluster(ctx, key, crunchybridgecluster)
 	if err != nil {
-		return ctrl.Result{}, err
+		return ctrl.Result{}, tracing.Escape(span, err)
 	}
 
 	// Get Cluster Status
 	err = r.handleGetClusterStatus(ctx, key, crunchybridgecluster)
 	if err != nil {
-		return ctrl.Result{}, err
+		return ctrl.Result{}, tracing.Escape(span, err)
 	}
 
 	// Get Cluster Upgrade
 	err = r.handleGetClusterUpgrade(ctx, key, crunchybridgecluster)
 	if err != nil {
-		return ctrl.Result{}, err
+		return ctrl.Result{}, tracing.Escape(span, err)
 	}
 
 	// Reconcile roles and their secrets
 	err = r.reconcilePostgresRoles(ctx, key, crunchybridgecluster)
 	if err != nil {
 		log.Error(err, "issue reconciling postgres user roles/secrets")
-		return ctrl.Result{}, err
+		return ctrl.Result{}, tracing.Escape(span, err)
 	}
 
 	// For now, we skip updating until the upgrade status is cleared.
diff --git a/internal/controller/pgupgrade/pgupgrade_controller.go b/internal/controller/pgupgrade/pgupgrade_controller.go
index c66494c6be..349a01ee89 100644
--- a/internal/controller/pgupgrade/pgupgrade_controller.go
+++ b/internal/controller/pgupgrade/pgupgrade_controller.go
@@ -98,6 +98,7 @@ func (r *PGUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	ctx, span := tracing.Start(ctx, "reconcile-pgupgrade")
 	log := logging.FromContext(ctx)
 	defer span.End()
+	defer func(s tracing.Span) { _ = tracing.Escape(s, err) }(span)
 
 	// Retrieve the upgrade from the client cache, if it exists. A deferred
 	// function below will send any changes to its Status field.
diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go
index 512738621c..933b781815 100644
--- a/internal/controller/postgrescluster/controller.go
+++ b/internal/controller/postgrescluster/controller.go
@@ -80,9 +80,8 @@ func (r *Reconciler) Reconcile(
 		// cluster is deleted.
 		if err = client.IgnoreNotFound(err); err != nil {
 			log.Error(err, "unable to fetch PostgresCluster")
-			span.RecordError(err)
 		}
-		return runtime.ErrorWithBackoff(err)
+		return runtime.ErrorWithBackoff(tracing.Escape(span, err))
 	}
 
 	// Set any defaults that may not have been stored in the API. No DeepCopy
@@ -107,9 +106,8 @@ func (r *Reconciler) Reconcile(
 	// Check for and handle deletion of cluster. Return early if it is being
 	// deleted or there was an error.
 	if result, err := r.handleDelete(ctx, cluster); err != nil {
-		span.RecordError(err)
 		log.Error(err, "deleting")
-		return runtime.ErrorWithBackoff(err)
+		return runtime.ErrorWithBackoff(tracing.Escape(span, err))
 
 	} else if result != nil {
 		if log := log.V(1); log.Enabled() {
@@ -130,7 +128,7 @@ func (r *Reconciler) Reconcile(
 		// specifically allow reconciliation if the cluster is shutdown to
 		// facilitate upgrades, otherwise return
 		if !initialize.FromPointer(cluster.Spec.Shutdown) {
-			return runtime.ErrorWithBackoff(err)
+			return runtime.ErrorWithBackoff(tracing.Escape(span, err))
 		}
 	}
 	// Issue Warning Event if postgres version is EOL according to PostgreSQL:
@@ -154,7 +152,7 @@ func (r *Reconciler) Reconcile(
 		path := field.NewPath("spec", "standby")
 		err := field.Invalid(path, cluster.Name, "Standby requires a host or repoName to be enabled")
 		r.Recorder.Event(cluster, corev1.EventTypeWarning, "InvalidStandbyConfiguration", err.Error())
-		return runtime.ErrorWithBackoff(err)
+		return runtime.ErrorWithBackoff(tracing.Escape(span, err))
 	}
 
 	var (
@@ -208,7 +206,7 @@ func (r *Reconciler) Reconcile(
 
 			ObservedGeneration: cluster.GetGeneration(),
 		})
-		return runtime.ErrorWithBackoff(patchClusterStatus())
+		return runtime.ErrorWithBackoff(tracing.Escape(span, patchClusterStatus()))
 	} else {
 		meta.RemoveStatusCondition(&cluster.Status.Conditions, v1beta1.PostgresClusterProgressing)
 	}
@@ -228,7 +226,7 @@ func (r *Reconciler) Reconcile(
 
 				ObservedGeneration: cluster.GetGeneration(),
 			})
-			return runtime.ErrorWithBackoff(patchClusterStatus())
+			return runtime.ErrorWithBackoff(tracing.Escape(span, patchClusterStatus()))
 		} else {
 			meta.RemoveStatusCondition(&cluster.Status.Conditions, v1beta1.PostgresClusterProgressing)
 		}
@@ -259,7 +257,8 @@ func (r *Reconciler) Reconcile(
 		// return is no longer needed, and reconciliation can proceed normally.
 		returnEarly, err := r.reconcileDirMoveJobs(ctx, cluster)
 		if err != nil || returnEarly {
-			return runtime.ErrorWithBackoff(errors.Join(err, patchClusterStatus()))
+			return runtime.ErrorWithBackoff(tracing.Escape(span,
+				errors.Join(err, patchClusterStatus())))
 		}
 	}
 	if err == nil {
@@ -309,7 +308,7 @@ func (r *Reconciler) Reconcile(
 		// can proceed normally.
 		returnEarly, err := r.reconcileDataSource(ctx, cluster, instances, clusterVolumes, rootCA, backupsSpecFound)
 		if err != nil || returnEarly {
-			return runtime.ErrorWithBackoff(errors.Join(err, patchClusterStatus()))
+			return runtime.ErrorWithBackoff(tracing.Escape(span, errors.Join(err, patchClusterStatus())))
 		}
 	}
 	if err == nil {
@@ -401,7 +400,7 @@ func (r *Reconciler) Reconcile(
 
 	log.V(1).Info("reconciled cluster")
 
-	return result, errors.Join(err, patchClusterStatus())
+	return result, tracing.Escape(span, errors.Join(err, patchClusterStatus()))
 }
 
 // deleteControlled safely deletes object when it is controlled by cluster.
diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go
index 4dfa37559c..97cc2cdce5 100644
--- a/internal/controller/postgrescluster/instance.go
+++ b/internal/controller/postgrescluster/instance.go
@@ -807,8 +807,7 @@ func (r *Reconciler) rolloutInstance(
 			err = errors.New("unable to switchover")
 		}
 
-		span.RecordError(err)
-		return err
+		return tracing.Escape(span, err)
 	}
 
 	// When the cluster has only one instance for failover, perform a series of
@@ -840,8 +839,7 @@ func (r *Reconciler) rolloutInstance(
 			logging.FromContext(ctx).V(1).Info("attempted checkpoint",
 				"duration", elapsed, "stdout", stdout, "stderr", stderr)
 
-			span.RecordError(err)
-			return elapsed, err
+			return elapsed, tracing.Escape(span, err)
 		}
 
 		duration, err := checkpoint(ctx)
@@ -950,8 +948,7 @@ func (r *Reconciler) rolloutInstances(
 		}
 	}
 
-	span.RecordError(err)
-	return err
+	return tracing.Escape(span, err)
 }
 
 // scaleDownInstances removes extra instances from a cluster until it matches
@@ -1081,20 +1078,23 @@ func (r *Reconciler) scaleUpInstances(
 	// While there are fewer instances than specified, generate another empty one
 	// and append it.
 	for len(instances) < int(*set.Replicas) {
-		_, span := tracing.Start(ctx, "generate-instance-name")
-		next := naming.GenerateInstance(cluster, set)
-		// if there are any available instance names (as determined by observing any PVCs for the
-		// instance set that are not currently associated with an instance, e.g. in the event the
-		// instance STS was deleted), then reuse them instead of generating a new name
-		if len(availableInstanceNames) > 0 {
-			next.Name = availableInstanceNames[0]
-			availableInstanceNames = availableInstanceNames[1:]
-		} else {
-			for instanceNames.Has(next.Name) {
-				next = naming.GenerateInstance(cluster, set)
+		next := func() metav1.ObjectMeta {
+			_, span := tracing.Start(ctx, "generate-instance-name")
+			defer span.End()
+			n := naming.GenerateInstance(cluster, set)
+			// if there are any available instance names (as determined by observing any PVCs for the
+			// instance set that are not currently associated with an instance, e.g. in the event the
+			// instance STS was deleted), then reuse them instead of generating a new name
+			if len(availableInstanceNames) > 0 {
+				n.Name = availableInstanceNames[0]
+				availableInstanceNames = availableInstanceNames[1:]
+			} else {
+				for instanceNames.Has(n.Name) {
+					n = naming.GenerateInstance(cluster, set)
+				}
 			}
-		}
-		span.End()
+			return n
+		}()
 
 		instanceNames.Insert(next.Name)
 		instances = append(instances, &appsv1.StatefulSet{ObjectMeta: next})
diff --git a/internal/controller/standalone_pgadmin/controller.go b/internal/controller/standalone_pgadmin/controller.go
index 481231684f..7e1005900c 100644
--- a/internal/controller/standalone_pgadmin/controller.go
+++ b/internal/controller/standalone_pgadmin/controller.go
@@ -93,7 +93,7 @@ func (r *PGAdminReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		// NotFound cannot be fixed by requeuing so ignore it. During background
 		// deletion, we receive delete events from pgadmin's dependents after
 		// pgadmin is deleted.
-		return ctrl.Result{}, client.IgnoreNotFound(err)
+		return ctrl.Result{}, tracing.Escape(span, client.IgnoreNotFound(err))
 	}
 
 	// Write any changes to the pgadmin status on the way out.
@@ -148,7 +148,7 @@ func (r *PGAdminReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		log.V(1).Info("Reconciled pgAdmin")
 	}
 
-	return ctrl.Result{}, err
+	return ctrl.Result{}, tracing.Escape(span, err)
 }
 
 // The owner reference created by controllerutil.SetControllerReference blocks
diff --git a/internal/naming/dns.go b/internal/naming/dns.go
index 3d770bd2aa..3925bfe988 100644
--- a/internal/naming/dns.go
+++ b/internal/naming/dns.go
@@ -79,11 +79,10 @@ func KubernetesClusterDomain(ctx context.Context) string {
 	api := "kubernetes.default.svc"
 	cname, err := net.DefaultResolver.LookupCNAME(ctx, api)
 
-	if err == nil {
+	if tracing.Check(span, err) {
 		return strings.TrimPrefix(cname, api+".")
 	}
 
-	span.RecordError(err)
 	// The kubeadm default is "cluster.local" and is adequate when not running
 	// in an actual Kubernetes cluster.
 	return "cluster.local."
diff --git a/internal/tracing/errors.go b/internal/tracing/errors.go
new file mode 100644
index 0000000000..d0e00cf56c
--- /dev/null
+++ b/internal/tracing/errors.go
@@ -0,0 +1,34 @@
+// Copyright 2021 - 2024 Crunchy Data Solutions, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package tracing
+
+import (
+	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// Check returns true when err is nil. Otherwise, it adds err as an exception
+// event on s and returns false. If you intend to return err, consider using
+// [Escape] instead.
+//
+// See: https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans
+func Check(s Span, err error) bool {
+	if err == nil {
+		return true
+	}
+	if s.IsRecording() {
+		s.RecordError(err)
+	}
+	return false
+}
+
+// Escape adds non-nil err as an escaped exception event on s and returns err.
+// See: https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans
+func Escape(s Span, err error) error {
+	if err != nil && s.IsRecording() {
+		s.RecordError(err, trace.WithAttributes(semconv.ExceptionEscaped(true)))
+	}
+	return err
+}
diff --git a/internal/tracing/errors_test.go b/internal/tracing/errors_test.go
new file mode 100644
index 0000000000..4f8f6d1be5
--- /dev/null
+++ b/internal/tracing/errors_test.go
@@ -0,0 +1,94 @@
+// Copyright 2021 - 2024 Crunchy Data Solutions, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package tracing
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
+	"gotest.tools/v3/assert"
+)
+
+func TestCheck(t *testing.T) {
+	recorder := tracetest.NewSpanRecorder()
+	tracer := trace.NewTracerProvider(
+		trace.WithSpanProcessor(recorder),
+	).Tracer("")
+
+	{
+		_, span := tracer.Start(context.Background(), "")
+		assert.Assert(t, Check(span, nil))
+		span.End()
+
+		spans := recorder.Ended()
+		assert.Equal(t, len(spans), 1)
+		assert.Equal(t, len(spans[0].Events()), 0, "expected no events")
+	}
+
+	{
+		_, span := tracer.Start(context.Background(), "")
+		assert.Assert(t, !Check(span, errors.New("msg")))
+		span.End()
+
+		spans := recorder.Ended()
+		assert.Equal(t, len(spans), 2)
+		assert.Equal(t, len(spans[1].Events()), 1, "expected one event")
+
+		event := spans[1].Events()[0]
+		assert.Equal(t, event.Name, semconv.ExceptionEventName)
+
+		attrs := event.Attributes
+		assert.Equal(t, len(attrs), 2)
+		assert.Equal(t, string(attrs[0].Key), "exception.type")
+		assert.Equal(t, string(attrs[1].Key), "exception.message")
+		assert.Equal(t, attrs[0].Value.AsInterface(), "*errors.errorString")
+		assert.Equal(t, attrs[1].Value.AsInterface(), "msg")
+	}
+}
+
+func TestEscape(t *testing.T) {
+	recorder := tracetest.NewSpanRecorder()
+	tracer := trace.NewTracerProvider(
+		trace.WithSpanProcessor(recorder),
+	).Tracer("")
+
+	{
+		_, span := tracer.Start(context.Background(), "")
+		assert.NilError(t, Escape(span, nil))
+		span.End()
+
+		spans := recorder.Ended()
+		assert.Equal(t, len(spans), 1)
+		assert.Equal(t, len(spans[0].Events()), 0, "expected no events")
+	}
+
+	{
+		_, span := tracer.Start(context.Background(), "")
+		expected := errors.New("somesuch")
+		assert.Assert(t, errors.Is(Escape(span, expected), expected),
+			"expected to unwrap the original error")
+		span.End()
+
+		spans := recorder.Ended()
+		assert.Equal(t, len(spans), 2)
+		assert.Equal(t, len(spans[1].Events()), 1, "expected one event")
+
+		event := spans[1].Events()[0]
+		assert.Equal(t, event.Name, semconv.ExceptionEventName)
+
+		attrs := event.Attributes
+		assert.Equal(t, len(attrs), 3)
+		assert.Equal(t, string(attrs[0].Key), "exception.escaped")
+		assert.Equal(t, string(attrs[1].Key), "exception.type")
+		assert.Equal(t, string(attrs[2].Key), "exception.message")
+		assert.Equal(t, attrs[0].Value.AsInterface(), true)
+		assert.Equal(t, attrs[1].Value.AsInterface(), "*errors.errorString")
+		assert.Equal(t, attrs[2].Value.AsInterface(), "somesuch")
+	}
+}