🐛 properly trigger upgrades on version and klusterlet change

.github/workflows/dependency-review.yml

@@ -25,4 +25,4 @@ jobs:
         uses: actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493 # v4.1.3
 
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 # v4.7.1
+        uses: actions/dependency-review-action@bc41886e18ea39df68b1b1245f4184881938e050 # v4.7.2

fleetconfig-controller/api/v1alpha1/fleetconfig_types.go

@@ -458,6 +458,11 @@ type JoinedSpoke struct {
 	// +kubebuilder:default:={}
 	// +optional
 	EnabledAddons []string `json:"enabledAddons,omitempty"`
+
+	// KlusterletHash is a hash of spoke.klusterlet.values
+	// +kubebuilder:default:=""
+	// +optional
+	KlusterletHash string `json:"klusterletHash,omitempty"`
 }
 
 // GetName returns the name of the joined spoke cluster.

fleetconfig-controller/build/Dockerfile.base

@@ -38,7 +38,7 @@ ARG ARCH
 RUN apk update && apk add --no-cache bash curl
 
 # Install clusteradm
-ARG CLUSTERADM_VERSION=1.0.1
+ARG CLUSTERADM_VERSION=1.0.2
 RUN curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash -s -- ${CLUSTERADM_VERSION}
 
 ## Stage 3: Compress binaries with upx to reduce image size

fleetconfig-controller/build/Dockerfile.devspace

@@ -16,7 +16,7 @@ RUN apk add --no-cache bash curl python3 py3-pip
 RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
 # Install clusteradm
-ARG CLUSTERADM_VERSION=1.0.1
+ARG CLUSTERADM_VERSION=1.0.2
 RUN curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash -s -- ${CLUSTERADM_VERSION}
 
 # Install aws-iam-authenticator if building for EKS

fleetconfig-controller/build/Dockerfile.eks

@@ -38,7 +38,7 @@ ARG ARCH
 RUN apk update && apk add --no-cache bash curl
 
 # Install clusteradm
-ARG CLUSTERADM_VERSION=1.0.1
+ARG CLUSTERADM_VERSION=1.0.2
 RUN curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash -s -- ${CLUSTERADM_VERSION}
 
 # Install aws-iam-authenticator

fleetconfig-controller/build/Dockerfile.gke

@@ -37,7 +37,7 @@ ARG ARCH
 RUN apk update && apk add --no-cache bash curl
 
 # Install clusteradm
-ARG CLUSTERADM_VERSION=1.0.1
+ARG CLUSTERADM_VERSION=1.0.2
 RUN curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash -s -- ${CLUSTERADM_VERSION}
 
 ## Stage 3: Compress binaries with upx to reduce image size

fleetconfig-controller/charts/fleetconfig-controller/crds/fleetconfig.open-cluster-management.io_fleetconfigs.yaml

@@ -2691,6 +2691,10 @@ spec:
                       items:
                         type: string
                       type: array
+                    klusterletHash:
+                      default: ""
+                      description: KlusterletHash is a hash of spoke.klusterlet.values
+                      type: string
                     kubeconfig:
                       description: Kubeconfig details for the Spoke cluster.
                       properties:

fleetconfig-controller/go.mod

@@ -5,6 +5,7 @@ go 1.24.4
 require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/go-logr/logr v1.4.3
+	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/onsi/ginkgo/v2 v2.23.4
 	github.com/onsi/gomega v1.37.0
 	github.com/openshift/build-machinery-go v0.0.0-20250602125535-1b6d00b8c37c

fleetconfig-controller/go.sum

@@ -142,6 +142,8 @@ github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4
 github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
+github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/moby/buildkit v0.20.0 h1:aF5RujjQ310Pn6SLL/wQYIrSsPXy0sQ5KvWifwq1h8Y=

fleetconfig-controller/hack/.versions.env

@@ -1,4 +1,4 @@
 AWSIAMAUTH_VERSION=0.7.2
 CERT_MANAGER_VERSION=v1.18.1
-CLUSTERADM_VERSION=1.0.1
+CLUSTERADM_VERSION=1.0.2
 OCM_VERSION=1.0.0

fleetconfig-controller/internal/controller/v1alpha1/hub.go

@@ -243,12 +243,16 @@ func hubNeedsUpgrade(ctx context.Context, fc *v1alpha1.FleetConfig, operatorC *o
 	if err != nil {
 		return false, fmt.Errorf("failed to detect bundleVersion from clustermanager spec: %w", err)
 	}
+	desiredBundleVersion, err := version.Normalize(fc.Spec.Hub.ClusterManager.Source.BundleVersion)
+	if err != nil {
+		return false, err
+	}
 
 	logger.V(0).Info("found clustermanager bundleVersions",
 		"activeBundleVersion", activeBundleVersion,
-		"desiredBundleVersion", fc.Spec.Hub.ClusterManager.Source.BundleVersion,
+		"desiredBundleVersion", desiredBundleVersion,
 	)
-	return activeBundleVersion == fc.Spec.Hub.ClusterManager.Source.BundleVersion, nil
+	return activeBundleVersion != desiredBundleVersion, nil
 }
 
 // getClusterManager retrieves the ClusterManager resource from the Hub cluster

fleetconfig-controller/internal/controller/v1alpha1/spoke.go

@@ -27,6 +27,7 @@ import (
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/api/v1alpha1"
 	exec_utils "github.com/open-cluster-management-io/lab/fleetconfig-controller/internal/exec"
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/internal/file"
+	"github.com/open-cluster-management-io/lab/fleetconfig-controller/internal/hash"
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/internal/kube"
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/internal/version"
 	"github.com/open-cluster-management-io/lab/fleetconfig-controller/pkg/common"
@@ -78,8 +79,7 @@ func handleSpokes(ctx context.Context, kClient client.Client, fc *v1alpha1.Fleet
 		}
 	}
 
-	allEnabledAddons := make([][]string, len(fc.Spec.Spokes))
-	for i, spoke := range fc.Spec.Spokes {
+	for _, spoke := range fc.Spec.Spokes {
 		logger.V(0).Info("handleSpokes: reconciling spoke cluster", "name", spoke.Name)
 
 		// check if the spoke has already been joined to the hub
@@ -163,19 +163,23 @@ func handleSpokes(ctx context.Context, kClient client.Client, fc *v1alpha1.Fleet
 			logger.V(0).Info("handleSpokes: labeled ManagedCluster as hub-as-spoke", "name", spoke.Name)
 		}
 
-		// attempt an upgrade whenever the klusterlet's bundleVersion changes
-		upgrade, err := spokeNeedsUpgrade(ctx, kClient, spoke)
+		// attempt an upgrade whenever the klusterlet's bundleVersion or values change
+		currKlusterletHash, err := hash.ComputeHash(spoke.Klusterlet.Values)
+		if err != nil {
+			return fmt.Errorf("failed to compute hash of spoke %s klusterlet values: %w", spoke.Name, err)
+		}
+		upgrade, err := spokeNeedsUpgrade(ctx, kClient, spoke, fc.Status.JoinedSpokes, currKlusterletHash)
 		if err != nil {
 			return fmt.Errorf("failed to check if spoke cluster needs upgrade: %w", err)
 		}
+
 		if upgrade {
 			if err := upgradeSpoke(ctx, kClient, fc, spoke); err != nil {
 				return fmt.Errorf("failed to upgrade spoke cluster %s: %w", spoke.Name, err)
 			}
 		}
 
 		enabledAddons, err := handleSpokeAddons(ctx, addonClient, spoke, fc)
-		allEnabledAddons[i] = enabledAddons
 		if err != nil {
 			msg := fmt.Sprintf("failed to enable addons for spoke cluster %s: %s", spoke.Name, err.Error())
 			fc.SetConditions(true, v1alpha1.NewCondition(
@@ -189,27 +193,33 @@ func handleSpokes(ctx context.Context, kClient client.Client, fc *v1alpha1.Fleet
 				"AddonsEnabled", spoke.AddonEnableType(), metav1.ConditionTrue, metav1.ConditionTrue,
 			))
 		}
-	}
 
-	// Only spokes which are joined, are eligible to be unjoined
-	for i, spoke := range fc.Spec.Spokes {
-		joinedCondition := fc.GetCondition(spoke.JoinType())
-		if joinedCondition == nil || joinedCondition.Status != metav1.ConditionTrue {
-			continue
-		}
 		js := v1alpha1.JoinedSpoke{
 			Name:                    spoke.Name,
 			Kubeconfig:              spoke.Kubeconfig,
 			PurgeKlusterletOperator: spoke.Klusterlet.PurgeOperator,
-			EnabledAddons:           allEnabledAddons[i],
+			EnabledAddons:           enabledAddons,
+			KlusterletHash:          currKlusterletHash,
 		}
 		joinedSpokes = append(joinedSpokes, js)
+
 	}
+
 	fc.Status.JoinedSpokes = joinedSpokes
 
 	return nil
 }
 
+func getJoinedSpoke(js []v1alpha1.JoinedSpoke, spokeName string) (v1alpha1.JoinedSpoke, bool) {
+	i := slices.IndexFunc(js, func(s v1alpha1.JoinedSpoke) bool {
+		return spokeName == s.Name
+	})
+	if i == -1 {
+		return v1alpha1.JoinedSpoke{}, false
+	}
+	return js[i], true
+}
+
 func getJoinedCondition(managedCluster *clusterv1.ManagedCluster) *metav1.Condition {
 	if managedCluster == nil || managedCluster.Status.Conditions == nil {
 		return nil
@@ -420,11 +430,27 @@ func joinSpoke(ctx context.Context, kClient client.Client, fc *v1alpha1.FleetCon
 	return nil
 }
 
-// spokeNeedsUpgrade checks if the klusterlet on a Spoke cluster has the desired bundle version
-func spokeNeedsUpgrade(ctx context.Context, kClient client.Client, spoke v1alpha1.Spoke) (bool, error) {
+// spokeNeedsUpgrade checks if the klusterlet on a Spoke cluster requires an upgrade. Upgrades are required when any of the following are true:
+//   - The bundle version in the spec does not match the klusterlet's active bundle version
+//   - The hash of the klusterlet chart values in the spec does not match the hash of the last applied klusterlet chart values
+func spokeNeedsUpgrade(ctx context.Context, kClient client.Client, spoke v1alpha1.Spoke, joinedSpokes []v1alpha1.JoinedSpoke, currKlusterletHash string) (bool, error) {
 	logger := log.FromContext(ctx)
 	logger.V(0).Info("spokeNeedsUpgrade", "spokeClusterName", spoke.Name)
 
+	hashChanged := false
+	prevJs, found := getJoinedSpoke(joinedSpokes, spoke.Name)
+	if found {
+		hashChanged = prevJs.KlusterletHash != currKlusterletHash
+		logger.V(2).Info("comparing klusterlet values hash",
+			"spoke", spoke.Name,
+			"prevHash", prevJs.KlusterletHash,
+			"currHash", currKlusterletHash,
+		)
+	}
+	if hashChanged {
+		return true, nil
+	}
+
 	if spoke.Klusterlet.Source.BundleVersion == "default" {
 		logger.V(0).Info("klusterlet bundleVersion is default, skipping upgrade")
 		return false, nil
@@ -463,12 +489,16 @@ func spokeNeedsUpgrade(ctx context.Context, kClient client.Client, spoke v1alpha
 	if err != nil {
 		return false, fmt.Errorf("failed to detect bundleVersion from klusterlet spec: %w", err)
 	}
+	desiredBundleVersion, err := version.Normalize(spoke.Klusterlet.Source.BundleVersion)
+	if err != nil {
+		return false, err
+	}
 
 	logger.V(0).Info("found klusterlet bundleVersions",
 		"activeBundleVersion", activeBundleVersion,
-		"desiredBundleVersion", spoke.Klusterlet.Source.BundleVersion,
+		"desiredBundleVersion", desiredBundleVersion,
 	)
-	return activeBundleVersion == spoke.Klusterlet.Source.BundleVersion, nil
+	return activeBundleVersion != desiredBundleVersion, nil
 }
 
 // upgradeSpoke upgrades the Spoke cluster's klusterlet to the specified version

fleetconfig-controller/internal/hash/hash.go

@@ -0,0 +1,22 @@
+// Package hash provides hashing utilities.
+package hash
+
+import (
+	"strconv"
+
+	"github.com/mitchellh/hashstructure/v2"
+)
+
+// ComputeHash computes the hash value of an arbitrary object
+func ComputeHash(obj any) (string, error) {
+	opts := &hashstructure.HashOptions{
+		ZeroNil: true,
+	}
+	// compute a hash value of any object
+	hash, err := hashstructure.Hash(obj, hashstructure.FormatV2, opts)
+	if err != nil {
+		return "", err
+	}
+	hashStr := strconv.FormatUint(hash, 16)
+	return hashStr, nil
+}

fleetconfig-controller/internal/version/version.go

@@ -44,3 +44,12 @@ func LowestBundleVersion(ctx context.Context, bundleSpecs []string) (string, err
 
 	return semvers[0].String(), nil
 }
+
+// Normalize returns a semver string with the leading `v` prefix stripped off
+func Normalize(v string) (string, error) {
+	sv, err := semver.NewVersion(v)
+	if err != nil {
+		return "", err
+	}
+	return sv.String(), nil
+}

fleetconfig-controller/internal/version/version_test.go

@@ -50,3 +50,43 @@ func TestLowestBundleVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestNormalize(t *testing.T) {
+	tests := []struct {
+		name    string
+		version string
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "valid version with v prefix",
+			version: "v1.2.3",
+			want:    "1.2.3",
+			wantErr: false,
+		},
+		{
+			name:    "valid version without v prefix",
+			version: "1.2.3",
+			want:    "1.2.3",
+			wantErr: false,
+		},
+		{
+			name:    "invalid version string",
+			version: "invalid-version",
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Normalize(tt.version)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Normalize(%v) error = %v, wantErr %v", tt.version, err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Normalize(%v) = %v, want %v", tt.version, got, tt.want)
+			}
+		})
+	}
+}