feat: Speedup drift detection by keeping track of resource versions

pkg/controllers/kluctl_project.go

@@ -824,13 +824,14 @@ func (pt *preparedTarget) kluctlPokeImages(ctx context.Context, targetContext *k
 	return cmdResult, err
 }
 
-func (pt *preparedTarget) kluctlDiff(ctx context.Context, targetContext *kluctl_project.TargetContext, crId string, objectsHash string) (*result.CommandResult, error) {
+func (pt *preparedTarget) kluctlDiff(ctx context.Context, targetContext *kluctl_project.TargetContext, crId string, objectsHash string, resourceVersions map[k8s.ObjectRef]string) (*result.CommandResult, error) {
 	timer := prometheus.NewTimer(internal_metrics.NewKluctlDeploymentDuration(pt.pp.obj.ObjectMeta.Namespace, pt.pp.obj.ObjectMeta.Name, pt.pp.obj.Spec.DeployMode))
 	defer timer.ObserveDuration()
 	cmd := commands.NewDiffCommand(targetContext)
 	cmd.ForceApply = pt.pp.obj.Spec.ForceApply
 	cmd.ReplaceOnError = pt.pp.obj.Spec.ReplaceOnError
 	cmd.ForceReplaceOnError = pt.pp.obj.Spec.ForceReplaceOnError
+	cmd.SkipResourceVersions = resourceVersions
 
 	cmdResult, cmdErr := cmd.Run()
 	err := pt.addCommandResultInfo(ctx, cmdResult, crId, objectsHash)

pkg/controllers/kluctldeployment_controller.go

@@ -12,6 +12,7 @@ import (
 	"github.com/kluctl/kluctl/v2/pkg/kluctl_jinja2"
 	"github.com/kluctl/kluctl/v2/pkg/results"
 	"github.com/kluctl/kluctl/v2/pkg/status"
+	"github.com/kluctl/kluctl/v2/pkg/types/k8s"
 	"github.com/kluctl/kluctl/v2/pkg/types/result"
 	"github.com/kluctl/kluctl/v2/pkg/utils"
 	"github.com/kluctl/kluctl/v2/pkg/utils/flux_utils/meta"
@@ -32,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sync"
 	"time"
 )
 
@@ -48,6 +50,9 @@ type KluctlDeploymentReconciler struct {
 	SshPool *ssh_pool.SshPool
 
 	ResultStore results.ResultStore
+
+	mutex               sync.Mutex
+	resourceVersionsMap map[client.ObjectKey]map[k8s.ObjectRef]string
 }
 
 // KluctlDeploymentReconcilerOpts contains options for the BaseReconciler.
@@ -407,6 +412,8 @@ func (r *KluctlDeploymentReconciler) doReconcile(
 		if err != nil {
 			log.Error(err, "Failed to write deploy result")
 		}
+
+		r.updateResourceVersions(key, deployResult.Objects, nil)
 	}
 
 	if needValidate {
@@ -427,11 +434,16 @@ func (r *KluctlDeploymentReconciler) doReconcile(
 			return nil, patchErr
 		}
 
-		diffResult, cmdErr := pt.kluctlDiff(ctx, targetContext, reconcileId, objectsHash)
-		err = obj.Status.SetLastDriftDetectionResult(diffResult.BuildDriftDetectionResult(), cmdErr)
+		resourceVersions := r.getResourceVersions(key)
+
+		diffResult, cmdErr := pt.kluctlDiff(ctx, targetContext, reconcileId, objectsHash, resourceVersions)
+		driftDetectionResult := diffResult.BuildDriftDetectionResult()
+		err = obj.Status.SetLastDriftDetectionResult(driftDetectionResult, cmdErr)
 		if err != nil {
 			log.Error(err, "Failed to write drift detection result")
 		}
+
+		r.updateResourceVersions(key, diffResult.Objects, driftDetectionResult.Objects)
 	}
 
 	var ctrlResult ctrl.Result
@@ -460,6 +472,45 @@ func (r *KluctlDeploymentReconciler) doReconcile(
 	return &ctrlResult, nil
 }
 
+func (r *KluctlDeploymentReconciler) buildResourceVersionsMap(objects []result.ResultObject) map[k8s.ObjectRef]string {
+	m := map[k8s.ObjectRef]string{}
+	for _, o := range objects {
+		if o.Applied == nil && o.Remote == nil {
+			continue
+		}
+		ro := o.Applied
+		if ro == nil {
+			ro = o.Remote
+		}
+		s := ro.GetK8sResourceVersion()
+		if s == "" {
+			continue
+		}
+		m[o.Ref] = s
+	}
+	return m
+}
+
+func (r *KluctlDeploymentReconciler) getResourceVersions(key client.ObjectKey) map[k8s.ObjectRef]string {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	m, _ := r.resourceVersionsMap[key]
+	return m
+}
+
+func (r *KluctlDeploymentReconciler) updateResourceVersions(key client.ObjectKey, diffObjects []result.ResultObject, driftedObjects []result.DriftedObject) {
+	newMap := r.buildResourceVersionsMap(diffObjects)
+
+	// ignore versions from already drifted objects so that we re-diff them the next time
+	for _, o := range driftedObjects {
+		delete(newMap, o.Ref)
+	}
+
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	r.resourceVersionsMap[key] = newMap
+}
+
 func (r *KluctlDeploymentReconciler) buildFinalStatus(ctx context.Context, obj *kluctlv1.KluctlDeployment) (finalStatus string, reason string) {
 	log := ctrl.LoggerFrom(ctx)
 
@@ -600,6 +651,10 @@ func (r *KluctlDeploymentReconciler) nextValidateTime(obj *kluctlv1.KluctlDeploy
 func (r *KluctlDeploymentReconciler) finalize(ctx context.Context, obj *kluctlv1.KluctlDeployment) (ctrl.Result, error) {
 	r.doFinalize(ctx, obj)
 
+	r.mutex.Lock()
+	delete(r.resourceVersionsMap, client.ObjectKeyFromObject(obj))
+	r.mutex.Unlock()
+
 	// Remove our finalizer from the list and update it
 	patch := client.MergeFrom(obj.DeepCopy())
 	controllerutil.RemoveFinalizer(obj, kluctlv1.KluctlDeploymentFinalizer)

pkg/controllers/kluctldeployment_controller_setup.go

@@ -3,14 +3,18 @@ package controllers
 import (
 	"context"
 	kluctlv1 "github.com/kluctl/kluctl/v2/api/v1beta1"
+	"github.com/kluctl/kluctl/v2/pkg/types/k8s"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *KluctlDeploymentReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opts KluctlDeploymentReconcilerOpts) error {
+	r.resourceVersionsMap = map[client.ObjectKey]map[k8s.ObjectRef]string{}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: opts.Concurrency,

pkg/deployment/commands/diff.go

@@ -18,6 +18,8 @@ type DiffCommand struct {
 	IgnoreTags          bool
 	IgnoreLabels        bool
 	IgnoreAnnotations   bool
+
+	SkipResourceVersions map[k8s2.ObjectRef]string
 }
 
 func NewDiffCommand(targetCtx *kluctl_project.TargetContext) *DiffCommand {
@@ -41,12 +43,13 @@ func (cmd *DiffCommand) Run() (*result.CommandResult, error) {
 	}
 
 	o := &utils.ApplyUtilOptions{
-		ForceApply:          cmd.ForceApply,
-		ReplaceOnError:      cmd.ReplaceOnError,
-		ForceReplaceOnError: cmd.ForceReplaceOnError,
-		DryRun:              true,
-		AbortOnError:        false,
-		ReadinessTimeout:    0,
+		ForceApply:           cmd.ForceApply,
+		ReplaceOnError:       cmd.ReplaceOnError,
+		ForceReplaceOnError:  cmd.ForceReplaceOnError,
+		DryRun:               true,
+		AbortOnError:         false,
+		ReadinessTimeout:     0,
+		SkipResourceVersions: cmd.SkipResourceVersions,
 	}
 	au := utils.NewApplyDeploymentsUtil(cmd.targetCtx.SharedContext.Ctx, dew, ru, cmd.targetCtx.SharedContext.K, o)
 	au.ApplyDeployments(cmd.targetCtx.DeploymentCollection.Deployments)

pkg/deployment/utils/apply_utils.go

@@ -33,6 +33,8 @@ type ApplyUtilOptions struct {
 	AbortOnError        bool
 	ReadinessTimeout    time.Duration
 	NoWait              bool
+
+	SkipResourceVersions map[k8s2.ObjectRef]string
 }
 
 type ApplyUtil struct {
@@ -339,6 +341,16 @@ func (a *ApplyUtil) ApplyObject(x *uo.UnstructuredObject, replaced bool, hook bo
 
 	x = a.k.FixObjectForPatch(x)
 	remoteObject := a.ru.GetRemoteObject(ref)
+
+	if a.o.SkipResourceVersions != nil && remoteObject != nil {
+		remoteResourceVersion := remoteObject.GetK8sResourceVersion()
+		skipVersion, ok := a.o.SkipResourceVersions[ref]
+		if ok && skipVersion == remoteResourceVersion {
+			a.handleResult(remoteObject, hook)
+			return
+		}
+	}
+
 	var remoteNamespace *uo.UnstructuredObject
 	if ref.Namespace != "" {
 		remoteNamespace = a.ru.GetRemoteNamespace(ref.Namespace)

pkg/types/result/drift_detection_result.go

@@ -5,6 +5,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type DriftedObject struct {
+	BaseObject
+
+	LastResourceVersion string `json:"lastResourceVersion"`
+}
+
 type DriftDetectionResult struct {
 	Id                  string                `json:"id"`
 	ProjectKey          ProjectKey            `json:"projectKey"`
@@ -16,7 +22,7 @@ type DriftDetectionResult struct {
 
 	Warnings []DeploymentError `json:"warnings,omitempty"`
 	Errors   []DeploymentError `json:"errors,omitempty"`
-	Objects  []BaseObject      `json:"objects,omitempty"`
+	Objects  []DriftedObject   `json:"objects,omitempty"`
 }
 
 func (cr *CommandResult) BuildDriftDetectionResult() *DriftDetectionResult {
@@ -39,7 +45,19 @@ func (cr *CommandResult) BuildDriftDetectionResult() *DriftDetectionResult {
 		if !o.New && !o.Orphan && !o.Deleted && len(o.Changes) == 0 {
 			continue
 		}
-		ret.Objects = append(ret.Objects, o.BaseObject)
+		resourceVersion := ""
+		ro := o.Applied
+		if ro == nil {
+			ro = o.Remote
+		}
+		if ro != nil {
+			resourceVersion = ro.GetK8sResourceVersion()
+		}
+
+		ret.Objects = append(ret.Objects, DriftedObject{
+			BaseObject:          o.BaseObject,
+			LastResourceVersion: resourceVersion,
+		})
 	}
 
 	return ret
@@ -48,7 +66,7 @@ func (cr *CommandResult) BuildDriftDetectionResult() *DriftDetectionResult {
 func (dr *DriftDetectionResult) BuildShortMessage() string {
 	ret := ""
 
-	count := func(f func(o BaseObject) bool) int {
+	count := func(f func(o DriftedObject) bool) int {
 		cnt := 0
 		for _, o := range dr.Objects {
 			if f(o) {
@@ -68,17 +86,17 @@ func (dr *DriftDetectionResult) BuildShortMessage() string {
 		ret += fmt.Sprintf("%d %s", cnt, s)
 	}
 
-	countAndAdd := func(s string, cntFun func(o BaseObject) bool) {
+	countAndAdd := func(s string, cntFun func(o DriftedObject) bool) {
 		add(s, count(cntFun))
 	}
 
 	if len(dr.Objects) == 0 {
 		ret = "no drift"
 	} else {
-		countAndAdd("new", func(o BaseObject) bool { return o.New })
-		countAndAdd("chg", func(o BaseObject) bool { return len(o.Changes) != 0 })
-		countAndAdd("orp", func(o BaseObject) bool { return o.Orphan })
-		countAndAdd("del", func(o BaseObject) bool { return o.Deleted })
+		countAndAdd("new", func(o DriftedObject) bool { return o.New })
+		countAndAdd("chg", func(o DriftedObject) bool { return len(o.Changes) != 0 })
+		countAndAdd("orp", func(o DriftedObject) bool { return o.Orphan })
+		countAndAdd("del", func(o DriftedObject) bool { return o.Deleted })
 		add("err", len(dr.Errors))
 		add("wrn", len(dr.Warnings))
 	}