Add debug ability to the telemetry operator

components/telemetry-operator/controller/logparser/controller.go

@@ -19,6 +19,7 @@ package logparser
 import (
 	"context"
 	"fmt"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/overrides"
 
 	telemetryv1alpha1 "github.com/kyma-project/kyma/components/telemetry-operator/apis/telemetry/v1alpha1"
 	"github.com/kyma-project/kyma/components/telemetry-operator/internal/configchecksum"
@@ -32,8 +33,10 @@ import (
 const checksumAnnotationKey = "checksum/logparser-config"
 
 type Config struct {
-	ParsersConfigMap types.NamespacedName
-	DaemonSet        types.NamespacedName
+	ParsersConfigMap  types.NamespacedName
+	DaemonSet         types.NamespacedName
+	OverrideConfigMap types.NamespacedName
+	Overrides         overrides.Config
 }
 
 //go:generate mockery --name DaemonSetProber --filename daemon_set_prober.go
@@ -48,20 +51,22 @@ type DaemonSetAnnotator interface {
 
 type Reconciler struct {
 	client.Client
-	config    Config
-	prober    DaemonSetProber
-	annotator DaemonSetAnnotator
-	syncer    syncer
+	config       Config
+	prober       DaemonSetProber
+	annotator    DaemonSetAnnotator
+	syncer       syncer
+	globalConfig *overrides.Handler
 }
 
-func NewReconciler(client client.Client, config Config, prober DaemonSetProber, annotator DaemonSetAnnotator) *Reconciler {
+func NewReconciler(client client.Client, config Config, prober DaemonSetProber, annotator DaemonSetAnnotator, handler *overrides.Handler) *Reconciler {
 	var r Reconciler
 
 	r.Client = client
 	r.config = config
 	r.prober = prober
 	r.annotator = annotator
 	r.syncer = syncer{client, config}
+	r.globalConfig = handler
 
 	return &r
 }
@@ -70,6 +75,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	log := logf.FromContext(ctx)
 	log.V(1).Info("Reconciliation triggered")
 
+	overrideConfig, err := r.globalConfig.UpdateOverrideConfig(ctx, r.config.OverrideConfigMap)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if err := r.globalConfig.CheckGlobalConfig(overrideConfig.Global); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if overrideConfig.Logging.Paused {
+		log.V(1).Info("Skipping reconciliation of logparser as reconciliation is paused.")
+		return ctrl.Result{}, nil
+	}
+
 	var parser telemetryv1alpha1.LogParser
 	if err := r.Get(ctx, req.NamespacedName, &parser); err != nil {
 		return ctrl.Result{}, client.IgnoreNotFound(err)

components/telemetry-operator/controller/logpipeline/controller.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kyma-project/kyma/components/telemetry-operator/internal/configchecksum"
 	configbuilder "github.com/kyma-project/kyma/components/telemetry-operator/internal/fluentbit/config/builder"
 	utils "github.com/kyma-project/kyma/components/telemetry-operator/internal/kubernetes"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/overrides"
 	resources "github.com/kyma-project/kyma/components/telemetry-operator/internal/resources/logpipeline"
 	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
@@ -40,8 +41,10 @@ type Config struct {
 	SectionsConfigMap types.NamespacedName
 	FilesConfigMap    types.NamespacedName
 	EnvSecret         types.NamespacedName
+	OverrideConfigMap types.NamespacedName
 	PipelineDefaults  configbuilder.PipelineDefaults
 	ManageFluentBit   bool
+	Overrides         overrides.Config
 }
 
 //go:generate mockery --name DaemonSetProber --filename daemon_set_prober.go
@@ -62,9 +65,10 @@ type Reconciler struct {
 	allLogPipelines         prometheus.Gauge
 	unsupportedLogPipelines prometheus.Gauge
 	syncer                  syncer
+	globalConfig            overrides.GlobalConfigHandler
 }
 
-func NewReconciler(client client.Client, config Config, prober DaemonSetProber, annotator DaemonSetAnnotator) *Reconciler {
+func NewReconciler(client client.Client, config Config, prober DaemonSetProber, annotator DaemonSetAnnotator, handler *overrides.Handler) *Reconciler {
 	var r Reconciler
 	r.Client = client
 	r.config = config
@@ -74,6 +78,7 @@ func NewReconciler(client client.Client, config Config, prober DaemonSetProber,
 	r.unsupportedLogPipelines = prometheus.NewGauge(prometheus.GaugeOpts{Name: "telemetry_unsupported_logpipelines", Help: "Number of log pipelines with custom filters or outputs."})
 	metrics.Registry.MustRegister(r.allLogPipelines, r.unsupportedLogPipelines)
 	r.syncer = syncer{client, config}
+	r.globalConfig = handler
 
 	return &r
 }
@@ -82,6 +87,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	log := logf.FromContext(ctx)
 	log.V(1).Info("Reconciliation triggered")
 
+	overrideConfig, err := r.globalConfig.UpdateOverrideConfig(ctx, r.config.OverrideConfigMap)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if err := r.globalConfig.CheckGlobalConfig(overrideConfig.Global); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if overrideConfig.Logging.Paused {
+		log.V(1).Info("Skipping reconciliation of logpipeline as reconciliation is paused.")
+		return ctrl.Result{}, nil
+	}
+
 	if err := r.updateMetrics(ctx); err != nil {
 		log.Error(err, "Failed to get all LogPipelines while updating metrics")
 	}

components/telemetry-operator/controller/logpipeline/suite_test.go

@@ -18,6 +18,8 @@ package logpipeline
 
 import (
 	"context"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/logger"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/overrides"
 	"path/filepath"
 	"testing"
 
@@ -28,6 +30,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	zapLog "go.uber.org/zap"
 	"k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
@@ -55,6 +58,7 @@ var (
 		SectionsConfigMap: types.NamespacedName{Name: "test-telemetry-fluent-bit-sections", Namespace: "default"},
 		FilesConfigMap:    types.NamespacedName{Name: "test-telemetry-fluent-bit-files", Namespace: "default"},
 		EnvSecret:         types.NamespacedName{Name: "test-telemetry-fluent-bit-env", Namespace: "default"},
+		OverrideConfigMap: types.NamespacedName{Name: "override-config", Namespace: "default"},
 		PipelineDefaults: builder.PipelineDefaults{
 			InputTag:          "kube",
 			MemoryBufferLimit: "10M",
@@ -72,6 +76,8 @@ func TestAPIs(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	dynamicLoglevel := zapLog.NewAtomicLevel()
+	configureLogLevelOnFly := logger.NewLogReconfigurer(dynamicLoglevel)
 
 	ctx, cancel = context.WithCancel(context.TODO())
 
@@ -105,7 +111,9 @@ var _ = BeforeSuite(func() {
 	Expect(err).ToNot(HaveOccurred())
 
 	client := mgr.GetClient()
-	reconciler := NewReconciler(client, testConfig, &kubernetes.DaemonSetProber{Client: client}, &kubernetes.DaemonSetAnnotator{Client: client})
+	overrides := overrides.New(configureLogLevelOnFly, &kubernetes.ConfigmapProber{Client: client})
+
+	reconciler := NewReconciler(client, testConfig, &kubernetes.DaemonSetProber{Client: client}, &kubernetes.DaemonSetAnnotator{Client: client}, overrides)
 	err = reconciler.SetupWithManager(mgr)
 	Expect(err).ToNot(HaveOccurred())
 

components/telemetry-operator/controller/tracepipeline/controller.go

@@ -20,16 +20,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	telemetryv1alpha1 "github.com/kyma-project/kyma/components/telemetry-operator/apis/telemetry/v1alpha1"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/configchecksum"
+	utils "github.com/kyma-project/kyma/components/telemetry-operator/internal/kubernetes"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/overrides"
+	corev1 "k8s.io/api/core/v1"
 	"strings"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	telemetryv1alpha1 "github.com/kyma-project/kyma/components/telemetry-operator/apis/telemetry/v1alpha1"
-	"github.com/kyma-project/kyma/components/telemetry-operator/internal/configchecksum"
-	utils "github.com/kyma-project/kyma/components/telemetry-operator/internal/kubernetes"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -42,9 +42,11 @@ type Config struct {
 	CreateServiceMonitor bool
 	BaseName             string
 	Namespace            string
+	OverrideConfigMap    types.NamespacedName
 
 	Deployment DeploymentConfig
 	Service    ServiceConfig
+	Overrides  overrides.Config
 }
 
 type DeploymentConfig struct {
@@ -67,24 +69,39 @@ type DeploymentProber interface {
 
 type Reconciler struct {
 	client.Client
-	config Config
-	Scheme *runtime.Scheme
-	prober DeploymentProber
+	config           Config
+	Scheme           *runtime.Scheme
+	prober           DeploymentProber
+	overridesHandler overrides.GlobalConfigHandler
 }
 
-func NewReconciler(client client.Client, config Config, prober DeploymentProber, scheme *runtime.Scheme) *Reconciler {
+func NewReconciler(client client.Client, config Config, prober DeploymentProber, scheme *runtime.Scheme, handler *overrides.Handler) *Reconciler {
 	var r Reconciler
 	r.Client = client
 	r.config = config
 	r.Scheme = scheme
 	r.prober = prober
+	r.overridesHandler = handler
 	return &r
 }
 
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (reconcileResult ctrl.Result, reconcileErr error) {
-	logger := logf.FromContext(ctx)
+	log := logf.FromContext(ctx)
 
-	logger.V(1).Info("Reconciliation triggered")
+	log.V(1).Info("Reconciliation triggered")
+
+	overrideConfig, err := r.overridesHandler.UpdateOverrideConfig(ctx, r.config.OverrideConfigMap)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if err := r.overridesHandler.CheckGlobalConfig(overrideConfig.Global); err != nil {
+		return ctrl.Result{}, err
+	}
+	if overrideConfig.Tracing.Paused {
+		log.V(1).Info("Skipping reconciliation of tracepipeline as reconciliation is paused")
+		return ctrl.Result{}, nil
+	}
 
 	var tracePipeline telemetryv1alpha1.TracePipeline
 	if err := r.Get(ctx, req.NamespacedName, &tracePipeline); err != nil {

components/telemetry-operator/controller/tracepipeline/setup.go

@@ -7,6 +7,7 @@ import (
 	telemetryv1alpha1 "github.com/kyma-project/kyma/components/telemetry-operator/apis/telemetry/v1alpha1"
 	"github.com/kyma-project/kyma/components/telemetry-operator/internal/setup"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"

components/telemetry-operator/controller/tracepipeline/suite_test.go

@@ -16,14 +16,18 @@ package tracepipeline
 import (
 	"context"
 	"github.com/kyma-project/kyma/components/telemetry-operator/internal/kubernetes"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/logger"
+	"github.com/kyma-project/kyma/components/telemetry-operator/internal/overrides"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
 	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	ctrl "sigs.k8s.io/controller-runtime"
 
+	zapLog "go.uber.org/zap"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -48,6 +52,7 @@ var testConfig = Config{
 	CreateServiceMonitor: false,
 	BaseName:             "telemetry-trace-collector",
 	Namespace:            "kyma-system",
+	OverrideConfigMap:    types.NamespacedName{Name: "override-config", Namespace: "kyma-system"},
 	Deployment: DeploymentConfig{
 		Image:         "otel/opentelemetry-collector-contrib:0.60.0",
 		CPULimit:      resource.MustParse("1"),
@@ -68,6 +73,8 @@ func TestAPIs(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	dynamicLoglevel := zapLog.NewAtomicLevel()
+	configureLogLevelOnFly := logger.NewLogReconfigurer(dynamicLoglevel)
 
 	ctx, cancel = context.WithCancel(context.TODO())
 
@@ -101,8 +108,9 @@ var _ = BeforeSuite(func() {
 		LeaderElectionID:       "ddd7ef0b.kyma-project.io",
 	})
 	Expect(err).ToNot(HaveOccurred())
+	overrides := overrides.New(configureLogLevelOnFly, &kubernetes.ConfigmapProber{Client: k8sClient})
 
-	reconciler := NewReconciler(mgr.GetClient(), testConfig, &kubernetes.DeploymentProber{Client: k8sClient}, scheme.Scheme)
+	reconciler := NewReconciler(mgr.GetClient(), testConfig, &kubernetes.DeploymentProber{Client: k8sClient}, scheme.Scheme, overrides)
 	err = reconciler.SetupWithManager(mgr)
 	Expect(err).ToNot(HaveOccurred())