fix(operator): dynamically create tracers during reconciliation (#804)

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>

operator/controllers/common/otel_utils.go

@@ -38,6 +38,9 @@ var (
 type otelConfig struct {
 	TracerProvider *trace.TracerProvider
 	OtelExporter   *trace.SpanExporter
+
+	mtx     sync.RWMutex
+	tracers map[string]ITracer
 }
 
 // do not export this type to make it accessible only via the GetInstance method (i.e Singleton)
@@ -46,7 +49,9 @@ var otelInstance *otelConfig
 func GetOtelInstance() *otelConfig {
 	// initialize once
 	otelInitOnce.Do(func() {
-		otelInstance = &otelConfig{}
+		otelInstance = &otelConfig{
+			tracers: map[string]ITracer{},
+		}
 	})
 
 	return otelInstance
@@ -62,6 +67,7 @@ func (o *otelConfig) InitOtelCollector(otelCollectorUrl string) error {
 	otel.SetTracerProvider(o.TracerProvider)
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
 	o.OtelExporter = &otelExporter
+	o.cleanTracers()
 	logger.Info("Successfully initialized OTel collector")
 	return nil
 }
@@ -72,6 +78,21 @@ func (o *otelConfig) ShutDown() {
 	}
 }
 
+func (o *otelConfig) GetTracer(name string) ITracer {
+	o.mtx.Lock()
+	defer o.mtx.Unlock()
+	if o.tracers[name] == nil {
+		o.tracers[name] = otel.Tracer(name)
+	}
+	return o.tracers[name]
+}
+
+func (o *otelConfig) cleanTracers() {
+	o.mtx.Lock()
+	defer o.mtx.Unlock()
+	o.tracers = map[string]ITracer{}
+}
+
 func GetOTelTracerProviderOptions(oTelCollectorUrl string) ([]trace.TracerProviderOption, trace.SpanExporter, error) {
 	var tracerProviderOptions []trace.TracerProviderOption
 	var otelExporter trace.SpanExporter

operator/controllers/common/otel_utils_test.go

@@ -229,3 +229,14 @@ func TestSetUpKeptnTaskMeters_ErrorCase(t *testing.T) {
 	require.Nil(t, got.EvaluationCount)
 	require.Nil(t, got.EvaluationDuration)
 }
+
+func Test_otelConfig_GetTracer(t *testing.T) {
+	otelConfig := GetOtelInstance()
+
+	tracer := otelConfig.GetTracer("new-tracer")
+	require.NotNil(t, tracer)
+
+	otelConfig.cleanTracers()
+
+	require.Empty(t, otelConfig.tracers)
+}

operator/controllers/common/tracer.go

@@ -0,0 +1,11 @@
+package common
+
+import "go.opentelemetry.io/otel/trace"
+
+//go:generate moq -pkg fake -skip-ensure -out ./fake/tracer_mock.go . ITracer
+type ITracer = trace.Tracer
+
+//go:generate moq -pkg fake -skip-ensure -out ./fake/tracerfactory_mock.go . TracerFactory
+type TracerFactory interface {
+	GetTracer(name string) ITracer
+}

operator/controllers/lifecycle/keptnapp/controller.go

@@ -26,7 +26,6 @@ import (
 	"github.com/keptn/lifecycle-toolkit/operator/apis/lifecycle/v1alpha2/common"
 	controllercommon "github.com/keptn/lifecycle-toolkit/operator/controllers/common"
 	controllererrors "github.com/keptn/lifecycle-toolkit/operator/controllers/errors"
-	"github.com/keptn/lifecycle-toolkit/operator/controllers/lifecycle/interfaces"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/propagation"
@@ -43,14 +42,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
-// KeptnAppReconciler reconciles a KeptnApp object
+const traceComponentName = "keptn/operator/app"
 
+// KeptnAppReconciler reconciles a KeptnApp object
 type KeptnAppReconciler struct {
 	client.Client
-	Scheme   *runtime.Scheme
-	Recorder record.EventRecorder
-	Log      logr.Logger
-	Tracer   interfaces.ITracer
+	Scheme        *runtime.Scheme
+	Recorder      record.EventRecorder
+	Log           logr.Logger
+	TracerFactory controllercommon.TracerFactory
 }
 
 //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptnapps,verbs=get;list;watch;create;update;patch;delete
@@ -84,7 +84,7 @@ func (r *KeptnAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	traceContextCarrier := propagation.MapCarrier(app.Annotations)
 	ctx = otel.GetTextMapPropagator().Extract(ctx, traceContextCarrier)
 
-	ctx, span := r.Tracer.Start(ctx, "reconcile_app", trace.WithSpanKind(trace.SpanKindConsumer))
+	ctx, span := r.getTracer().Start(ctx, "reconcile_app", trace.WithSpanKind(trace.SpanKindConsumer))
 	defer span.End()
 
 	app.SetSpanAttributes(span)
@@ -138,10 +138,10 @@ func (r *KeptnAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 func (r *KeptnAppReconciler) createAppVersion(ctx context.Context, app *klcv1alpha2.KeptnApp) (*klcv1alpha2.KeptnAppVersion, error) {
-	ctx, span := r.Tracer.Start(ctx, "create_app_version", trace.WithSpanKind(trace.SpanKindProducer))
+	ctx, span := r.getTracer().Start(ctx, "create_app_version", trace.WithSpanKind(trace.SpanKindProducer))
 	defer span.End()
 
-	ctxAppTrace, spanAppTrace := r.Tracer.Start(ctx, app.GetAppVersionName(), trace.WithNewRoot(), trace.WithSpanKind(trace.SpanKindServer))
+	ctxAppTrace, spanAppTrace := r.getTracer().Start(ctx, app.GetAppVersionName(), trace.WithNewRoot(), trace.WithSpanKind(trace.SpanKindServer))
 	defer spanAppTrace.End()
 
 	app.SetSpanAttributes(span)
@@ -201,3 +201,7 @@ func (r *KeptnAppReconciler) deprecateAppVersions(ctx context.Context, app *klcv
 	}
 	return lastResultErr
 }
+
+func (r *KeptnAppReconciler) getTracer() controllercommon.ITracer {
+	return r.TracerFactory.GetTracer(traceComponentName)
+}

operator/controllers/lifecycle/keptnapp/controller_test.go

@@ -10,7 +10,6 @@ import (
 	apicommon "github.com/keptn/lifecycle-toolkit/operator/apis/lifecycle/v1alpha2/common"
 	controllercommon "github.com/keptn/lifecycle-toolkit/operator/controllers/common"
 	"github.com/keptn/lifecycle-toolkit/operator/controllers/common/fake"
-	interfacesfake "github.com/keptn/lifecycle-toolkit/operator/controllers/lifecycle/interfaces/fake"
 	"github.com/magiconair/properties/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel/trace"
@@ -165,27 +164,31 @@ func TestKeptnAppReconciler_deprecateAppVersions(t *testing.T) {
 	assert.Matches(t, event, `Normal CreateAppVersionAppVersionDeprecated Create AppVersion: deprecated KeptnAppVersions for KeptnAppVersion: myapp-1.0.0-2 / Namespace: default, Name: myapp, Version: 1.0.0`)
 }
 
-func setupReconciler() (*KeptnAppReconciler, chan string, *interfacesfake.ITracerMock) {
+func setupReconciler() (*KeptnAppReconciler, chan string, *fake.ITracerMock) {
 	//setup logger
 	opts := zap.Options{
 		Development: true,
 	}
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	//fake a tracer
-	tr := &interfacesfake.ITracerMock{StartFunc: func(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	tr := &fake.ITracerMock{StartFunc: func(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
 		return ctx, trace.SpanFromContext(ctx)
 	}}
 
+	tf := &fake.TracerFactoryMock{GetTracerFunc: func(name string) trace.Tracer {
+		return tr
+	}}
+
 	fakeClient := fake.NewClient()
 
 	recorder := record.NewFakeRecorder(100)
 	r := &KeptnAppReconciler{
-		Client:   fakeClient,
-		Scheme:   scheme.Scheme,
-		Recorder: recorder,
-		Log:      ctrl.Log.WithName("test-appController"),
-		Tracer:   tr,
+		Client:        fakeClient,
+		Scheme:        scheme.Scheme,
+		Recorder:      recorder,
+		Log:           ctrl.Log.WithName("test-appController"),
+		TracerFactory: tf,
 	}
 	return r, recorder.Events, tr
 }

operator/controllers/lifecycle/keptnappversion/controller.go

@@ -26,7 +26,6 @@ import (
 	apicommon "github.com/keptn/lifecycle-toolkit/operator/apis/lifecycle/v1alpha2/common"
 	controllercommon "github.com/keptn/lifecycle-toolkit/operator/controllers/common"
 	controllererrors "github.com/keptn/lifecycle-toolkit/operator/controllers/errors"
-	"github.com/keptn/lifecycle-toolkit/operator/controllers/lifecycle/interfaces"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/propagation"
@@ -41,15 +40,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const traceComponentName = "keptn/operator/appversion"
+
 // KeptnAppVersionReconciler reconciles a KeptnAppVersion object
 type KeptnAppVersionReconciler struct {
 	Scheme *runtime.Scheme
 	client.Client
-	Log         logr.Logger
-	Recorder    record.EventRecorder
-	Tracer      interfaces.ITracer
-	Meters      apicommon.KeptnMeters
-	SpanHandler controllercommon.ISpanHandler
+	Log           logr.Logger
+	Recorder      record.EventRecorder
+	TracerFactory controllercommon.TracerFactory
+	Meters        apicommon.KeptnMeters
+	SpanHandler   controllercommon.ISpanHandler
 }
 
 //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptnappversions,verbs=get;list;watch;create;update;patch;delete
@@ -94,7 +95,7 @@ func (r *KeptnAppVersionReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		SpanHandler: r.SpanHandler,
 	}
 
-	ctxAppTrace, spanAppTrace, err := r.SpanHandler.GetSpan(ctxAppTrace, r.Tracer, appVersion, "")
+	ctxAppTrace, spanAppTrace, err := r.SpanHandler.GetSpan(ctxAppTrace, r.getTracer(), appVersion, "")
 	if err != nil {
 		r.Log.Error(err, "could not get span")
 	}
@@ -109,7 +110,7 @@ func (r *KeptnAppVersionReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		reconcilePreDep := func(phaseCtx context.Context) (apicommon.KeptnState, error) {
 			return r.reconcilePrePostDeployment(ctx, phaseCtx, appVersion, apicommon.PreDeploymentCheckType)
 		}
-		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.Tracer, appVersion, phase, span, reconcilePreDep)
+		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.getTracer(), appVersion, phase, span, reconcilePreDep)
 		if !result.Continue {
 			return result.Result, err
 		}
@@ -120,7 +121,7 @@ func (r *KeptnAppVersionReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		reconcilePreEval := func(phaseCtx context.Context) (apicommon.KeptnState, error) {
 			return r.reconcilePrePostEvaluation(ctx, phaseCtx, appVersion, apicommon.PreDeploymentEvaluationCheckType)
 		}
-		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.Tracer, appVersion, phase, span, reconcilePreEval)
+		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.getTracer(), appVersion, phase, span, reconcilePreEval)
 		if !result.Continue {
 			return result.Result, err
 		}
@@ -131,7 +132,7 @@ func (r *KeptnAppVersionReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		reconcileAppDep := func(phaseCtx context.Context) (apicommon.KeptnState, error) {
 			return r.reconcileWorkloads(ctx, appVersion)
 		}
-		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.Tracer, appVersion, phase, span, reconcileAppDep)
+		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.getTracer(), appVersion, phase, span, reconcileAppDep)
 		if !result.Continue {
 			return result.Result, err
 		}
@@ -142,7 +143,7 @@ func (r *KeptnAppVersionReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		reconcilePostDep := func(phaseCtx context.Context) (apicommon.KeptnState, error) {
 			return r.reconcilePrePostDeployment(ctx, phaseCtx, appVersion, apicommon.PostDeploymentCheckType)
 		}
-		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.Tracer, appVersion, phase, span, reconcilePostDep)
+		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.getTracer(), appVersion, phase, span, reconcilePostDep)
 		if !result.Continue {
 			return result.Result, err
 		}
@@ -153,7 +154,7 @@ func (r *KeptnAppVersionReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		reconcilePostEval := func(phaseCtx context.Context) (apicommon.KeptnState, error) {
 			return r.reconcilePrePostEvaluation(ctx, phaseCtx, appVersion, apicommon.PostDeploymentEvaluationCheckType)
 		}
-		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.Tracer, appVersion, phase, span, reconcilePostEval)
+		result, err := phaseHandler.HandlePhase(ctx, ctxAppTrace, r.getTracer(), appVersion, phase, span, reconcilePostEval)
 		if !result.Continue {
 			return result.Result, err
 		}
@@ -208,7 +209,7 @@ func (r *KeptnAppVersionReconciler) setupSpansContexts(ctx context.Context, appV
 	appTraceContextCarrier := propagation.MapCarrier(appVersion.Spec.TraceId)
 	ctxAppTrace := otel.GetTextMapPropagator().Extract(context.TODO(), appTraceContextCarrier)
 
-	ctx, span := r.Tracer.Start(ctx, "reconcile_app_version", trace.WithSpanKind(trace.SpanKindConsumer))
+	ctx, span := r.getTracer().Start(ctx, "reconcile_app_version", trace.WithSpanKind(trace.SpanKindConsumer))
 
 	endFunc := func() {
 		if appVersion.IsEndTimeSet() {
@@ -229,3 +230,7 @@ func (r *KeptnAppVersionReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&klcv1alpha2.KeptnAppVersion{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
 		Complete(r)
 }
+
+func (r *KeptnAppVersionReconciler) getTracer() trace.Tracer {
+	return r.TracerFactory.GetTracer(traceComponentName)
+}