feat(operator): Introduce OTel tracing for Task controller (#128)

Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com>
keptn · Oct 7, 2022 · 0baf7a9 · 0baf7a9
1 parent 60651d1
commit 0baf7a9
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 0 deletions.
diff --git a/operator/api/v1alpha1/semconv/semconv.go b/operator/api/v1alpha1/semconv/semconv.go
@@ -18,6 +18,14 @@ func AddAttributeFromWorkloadInstance(s trace.Span, w v1alpha1.KeptnWorkloadInst
 	s.SetAttributes(common.Version.String(w.Spec.Version))
 }
 
+func AddAttributeFromTask(s trace.Span, t v1alpha1.KeptnTask) {
+	s.SetAttributes(common.ApplicationName.String(t.Spec.AppName))
+	s.SetAttributes(common.Workload.String(t.Spec.Workload))
+	s.SetAttributes(common.Version.String(t.Spec.WorkloadVersion))
+	s.SetAttributes(common.TaskName.String(t.Name))
+	s.SetAttributes(common.TaskType.String(string(t.Spec.Type)))
+}
+
 func AddAttributeFromAnnotations(s trace.Span, annotations map[string]string) {
 	s.SetAttributes(common.ApplicationName.String(annotations[common.AppAnnotation]))
 	s.SetAttributes(common.Workload.String(annotations[common.WorkloadAnnotation]))

diff --git a/operator/controllers/keptntask/controller.go b/operator/controllers/keptntask/controller.go
@@ -24,6 +24,11 @@ import (
 	"github.com/go-logr/logr"
 	klcv1alpha1 "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1"
 	"github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/common"
+	"github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/semconv"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 	batchv1 "k8s.io/api/batch/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -41,6 +46,7 @@ type KeptnTaskReconciler struct {
 	Recorder record.EventRecorder
 	Log      logr.Logger
 	Meters   common.KeptnMeters
+	Tracer   trace.Tracer
 }
 
 //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptntasks,verbs=get;list;watch;create;update;patch;delete
@@ -63,6 +69,14 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
 	}
 
+	traceContextCarrier := propagation.MapCarrier(task.Annotations)
+	ctx = otel.GetTextMapPropagator().Extract(ctx, traceContextCarrier)
+
+	ctx, span := r.Tracer.Start(ctx, "reconcile_task", trace.WithSpanKind(trace.SpanKindConsumer))
+	defer span.End()
+
+	semconv.AddAttributeFromTask(span, *task)
+
 	if !task.IsStartTimeSet() {
 		// metrics: increment active task counter
 		r.Meters.TaskActive.Add(ctx, 1, task.GetActiveMetricsAttributes()...)
@@ -71,18 +85,21 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	err := r.Client.Status().Update(ctx, task)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return ctrl.Result{Requeue: true}, err
 	}
 
 	jobExists, err := r.JobExists(ctx, *task, req.Namespace)
 	if err != nil {
 		r.Log.Error(err, "Could not check if job is running")
+		span.SetStatus(codes.Error, err.Error())
 		return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
 	}
 
 	if !jobExists {
 		err = r.createJob(ctx, req, task)
 		if err != nil {
+			span.SetStatus(codes.Error, err.Error())
 			return ctrl.Result{Requeue: true}, err
 		}
 		return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil
@@ -91,6 +108,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	if !task.Status.Status.IsCompleted() {
 		err := r.updateJob(ctx, req, task)
 		if err != nil {
+			span.SetStatus(codes.Error, err.Error())
 			return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, err
 		}
 		return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil
@@ -108,6 +126,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	err = r.Client.Status().Update(ctx, task)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return ctrl.Result{Requeue: true}, err
 	}
 

diff --git a/operator/main.go b/operator/main.go
@@ -203,6 +203,7 @@ func main() {
 		Log:      ctrl.Log.WithName("KeptnTask Controller"),
 		Recorder: mgr.GetEventRecorderFor("keptntask-controller"),
 		Meters:   meters,
+		Tracer:   otel.Tracer("keptn/operator/task"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "KeptnTask")
 		os.Exit(1)