diff --git a/assets/controller.yaml b/assets/controller.yaml index fe5a1ef8..dd688962 100644 --- a/assets/controller.yaml +++ b/assets/controller.yaml @@ -53,6 +53,7 @@ spec: - --output-file-path=/etc/merged-cloud-config/cloud.conf # Force disable node's managed identity, azure-disk-credentials Secret should be used. - --disable-identity-extension-auth + - --enable-azure-workload-identity=${ENABLE_AZURE_WORKLOAD_IDENTITY} env: - name: AZURE_CLIENT_ID valueFrom: @@ -64,6 +65,19 @@ spec: secretKeyRef: name: azure-disk-credentials key: azure_client_secret + optional: true + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_tenant_id + optional: true + - name: AZURE_FEDERATED_TOKEN_FILE + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_federated_token_file + optional: true volumeMounts: - name: host-cloud-config mountPath: /etc/cloud-config @@ -109,6 +123,9 @@ spec: - name: msi mountPath: /var/lib/waagent/ManagedIdentity-Settings readOnly: true + - name: bound-sa-token + mountPath: /var/run/secrets/openshift/serviceaccount + readOnly: true resources: requests: memory: 50Mi @@ -352,3 +369,9 @@ spec: secretName: azure-disk-csi-driver-controller-metrics-serving-cert - name: merged-cloud-config emptydir: + - name: bound-sa-token + projected: + sources: + - serviceAccountToken: + path: token + audience: openshift diff --git a/assets/node.yaml b/assets/node.yaml index 6f6801be..797cd17c 100644 --- a/assets/node.yaml +++ b/assets/node.yaml @@ -39,6 +39,7 @@ spec: - --output-file-path=/etc/merged-cloud-config/cloud.conf # Force disable node's managed identity, azure-disk-credentials Secret should be used. - --disable-identity-extension-auth + - --enable-azure-workload-identity=${ENABLE_AZURE_WORKLOAD_IDENTITY} env: - name: AZURE_CLIENT_ID valueFrom: @@ -50,6 +51,19 @@ spec: secretKeyRef: name: azure-disk-credentials key: azure_client_secret + optional: true + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_tenant_id + optional: true + - name: AZURE_FEDERATED_TOKEN_FILE + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_federated_token_file + optional: true volumeMounts: - name: host-cloud-config mountPath: /etc/cloud-config diff --git a/pkg/azurestackhub/azure_stack_hub_test.go b/pkg/azurestackhub/azure_stack_hub_test.go index 6ff06078..1142b1bd 100644 --- a/pkg/azurestackhub/azure_stack_hub_test.go +++ b/pkg/azurestackhub/azure_stack_hub_test.go @@ -41,7 +41,7 @@ func TestInjectPodSpecHappyPath(t *testing.T) { assert.Nil(t, yaml.Unmarshal(file, dep)) injectEnvAndMounts(&dep.Spec.Template.Spec) - assert.Len(t, dep.Spec.Template.Spec.Volumes, 6) + assert.Len(t, dep.Spec.Template.Spec.Volumes, 7) foundCfgVolume := false for _, v := range dep.Spec.Template.Spec.Volumes { if v.Name == azureCfgName { @@ -59,7 +59,7 @@ func TestInjectPodSpecHappyPath(t *testing.T) { } } assert.NotNil(t, csiDriver, "no csi-driver container found") - assert.Len(t, csiDriver.VolumeMounts, 4) + assert.Len(t, csiDriver.VolumeMounts, 5) foundCfgVolumeMount := false for _, v := range csiDriver.VolumeMounts { if v.Name == azureCfgName { diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index bd5bb177..340d1fff 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -7,8 +7,6 @@ import ( "strings" "time" - "github.com/openshift/azure-disk-csi-driver-operator/pkg/azurestackhub" - apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" @@ -16,14 +14,17 @@ import ( "k8s.io/client-go/rest" "k8s.io/klog/v2" + configv1 "github.com/openshift/api/config/v1" opv1 "github.com/openshift/api/operator/v1" "github.com/openshift/azure-disk-csi-driver-operator/assets" + "github.com/openshift/azure-disk-csi-driver-operator/pkg/azurestackhub" configclient "github.com/openshift/client-go/config/clientset/versioned" configinformers "github.com/openshift/client-go/config/informers/externalversions" opclient "github.com/openshift/client-go/operator/clientset/versioned" opinformers "github.com/openshift/client-go/operator/informers/externalversions" "github.com/openshift/library-go/pkg/controller/controllercmd" "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" "github.com/openshift/library-go/pkg/operator/csi/csicontrollerset" "github.com/openshift/library-go/pkg/operator/csi/csidrivercontrollerservicecontroller" "github.com/openshift/library-go/pkg/operator/csi/csidrivernodeservicecontroller" @@ -37,11 +38,13 @@ const ( operandName = "azure-disk-csi-driver" openShiftConfigNamespace = "openshift-config" secretName = "azure-disk-credentials" + tokenFileKey = "azure_federated_token_file" trustedCAConfigMap = "azure-disk-csi-driver-trusted-ca-bundle" resync = 20 * time.Minute - ccmOperatorImageEnvName = "CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE" - diskEncryptionSetID = "diskEncryptionSetID" + ccmOperatorImageEnvName = "CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE" + diskEncryptionSetID = "diskEncryptionSetID" + operatorImageVersionEnvVarName = "OPERATOR_IMAGE_VERSION" ) func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error { @@ -101,6 +104,32 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller go azureStackConfigSyncer.Run(ctx, 1) } + desiredVersion := os.Getenv(operatorImageVersionEnvVarName) + missingVersion := "0.0.1-snapshot" + + featureGateAccessor := featuregates.NewFeatureGateAccess( + desiredVersion, + missingVersion, + configInformers.Config().V1().ClusterVersions(), + configInformers.Config().V1().FeatureGates(), + controllerConfig.EventRecorder, + ) + go featureGateAccessor.Run(ctx) + go configInformers.Start(ctx.Done()) + + select { + case <-featureGateAccessor.InitialFeatureGatesObserved(): + featureGates, _ := featureGateAccessor.CurrentFeatureGates() + klog.Info("FeatureGates initialized", "knownFeatures", featureGates.KnownFeatures()) + case <-time.After(1 * time.Minute): + klog.Error(nil, "timed out waiting for FeatureGate detection") + return fmt.Errorf("timed out waiting for FeatureGate detection") + } + + replacedAssets := &assetWithReplacement{} + replacedAssets.Replace("${CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE}", os.Getenv(ccmOperatorImageEnvName)) + replaceWorkloadIdentityConfig(replacedAssets, featureGateAccessor, kubeClient) + csiControllerSet := csicontrollerset.NewCSIControllerSet( operatorClient, controllerConfig.EventRecorder, @@ -160,7 +189,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller configInformers, ).WithCSIDriverControllerService( "AzureDiskDriverControllerServiceController", - assetWithImageReplaced(), + replacedAssets.GetAssetFunc(), "controller.yaml", kubeClient, kubeInformersForNamespaces.InformersFor(defaultNamespace), @@ -181,7 +210,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller csidrivercontrollerservicecontroller.WithSecretHashAnnotationHook(defaultNamespace, secretName, secretInformer), ).WithCSIDriverNodeService( "AzureDiskDriverNodeServiceController", - assetWithImageReplaced(), + replacedAssets.GetAssetFunc(), "node.yaml", kubeClient, kubeInformersForNamespaces.InformersFor(defaultNamespace), @@ -227,14 +256,54 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller return fmt.Errorf("stopped") } -func assetWithImageReplaced() func(name string) ([]byte, error) { +type assetWithReplacement []string + +func (r *assetWithReplacement) Replace(old, new string) { + *r = append(*r, old, new) +} + +func (r *assetWithReplacement) GetAssetFunc() func(name string) ([]byte, error) { return func(name string) ([]byte, error) { assetBytes, err := assets.ReadFile(name) if err != nil { return assetBytes, err } - asset := string(assetBytes) - asset = strings.ReplaceAll(asset, "${CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE}", os.Getenv(ccmOperatorImageEnvName)) + + replacer := strings.NewReplacer(*r...) + asset := replacer.Replace(string(assetBytes)) + return []byte(asset), nil } } + +func replaceWorkloadIdentityConfig(assets *assetWithReplacement, fg featuregates.FeatureGateAccess, kubeClient *kubeclient.Clientset) error { + featureGates, err := fg.CurrentFeatureGates() + if err != nil { + return err + } + wiEnabled, err := isWorkloadIdentityEnabled(featureGates, kubeClient) + if err != nil { + return err + } + if wiEnabled { + assets.Replace("${ENABLE_AZURE_WORKLOAD_IDENTITY}", "true") + } else { + assets.Replace("${ENABLE_AZURE_WORKLOAD_IDENTITY}", "false") + } + return nil +} + +func isWorkloadIdentityEnabled(featureGates featuregates.FeatureGate, kubeClient *kubeclient.Clientset) (bool, error) { + if !featureGates.Enabled(configv1.FeatureGateAzureWorkloadIdentity) { + return false, nil + } + secret, err := kubeClient.CoreV1().Secrets(defaultNamespace).Get(context.Background(), secretName, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("could not get secret %s/%s: %v", defaultNamespace, secretName, err) + } + _, hasKey := secret.Data[tokenFileKey] + if !hasKey { + klog.Warningf("Workloads Identity feature will be disabled: feature gate is enabled, but secret %s/%s doesn't have the %q key.", defaultNamespace, secretName, tokenFileKey) + } + return hasKey, nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/featuregate.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/featuregate.go new file mode 100644 index 00000000..5ff0f3af --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/featuregate.go @@ -0,0 +1,47 @@ +package featuregates + +import ( + "fmt" + configv1 "github.com/openshift/api/config/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +// FeatureGate indicates whether a given feature is enabled or not +// This interface is heavily influenced by k8s.io/component-base, but not exactly compatible. +type FeatureGate interface { + // Enabled returns true if the key is enabled. + Enabled(key configv1.FeatureGateName) bool + // KnownFeatures returns a slice of strings describing the FeatureGate's known features. + KnownFeatures() []configv1.FeatureGateName +} + +type featureGate struct { + enabled sets.Set[configv1.FeatureGateName] + disabled sets.Set[configv1.FeatureGateName] +} + +func NewFeatureGate(enabled, disabled []configv1.FeatureGateName) FeatureGate { + return &featureGate{ + enabled: sets.New[configv1.FeatureGateName](enabled...), + disabled: sets.New[configv1.FeatureGateName](disabled...), + } +} + +func (f *featureGate) Enabled(key configv1.FeatureGateName) bool { + if f.enabled.Has(key) { + return true + } + if f.disabled.Has(key) { + return false + } + + panic(fmt.Errorf("feature %q is not registered in FeatureGates %v", key, f.KnownFeatures())) +} + +func (f *featureGate) KnownFeatures() []configv1.FeatureGateName { + allKnown := sets.NewString() + allKnown.Insert(FeatureGateNamesToStrings(f.enabled.UnsortedList())...) + allKnown.Insert(FeatureGateNamesToStrings(f.disabled.UnsortedList())...) + + return StringsToFeatureGateNames(allKnown.List()) +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/hardcoded_featuregate_reader.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/hardcoded_featuregate_reader.go new file mode 100644 index 00000000..58ae7176 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/hardcoded_featuregate_reader.go @@ -0,0 +1,78 @@ +package featuregates + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" +) + +type hardcodedFeatureGateAccess struct { + enabled []configv1.FeatureGateName + disabled []configv1.FeatureGateName + readErr error + + initialFeatureGatesObserved chan struct{} +} + +// NewHardcodedFeatureGateAccess returns a FeatureGateAccess that is always initialized and always +// returns the provided feature gates. +func NewHardcodedFeatureGateAccess(enabled, disabled []configv1.FeatureGateName) FeatureGateAccess { + initialFeatureGatesObserved := make(chan struct{}) + close(initialFeatureGatesObserved) + c := &hardcodedFeatureGateAccess{ + enabled: enabled, + disabled: disabled, + initialFeatureGatesObserved: initialFeatureGatesObserved, + } + + return c +} + +// NewHardcodedFeatureGateAccessForTesting returns a FeatureGateAccess that returns stub responses +// using caller-supplied values. +func NewHardcodedFeatureGateAccessForTesting(enabled, disabled []configv1.FeatureGateName, initialFeatureGatesObserved chan struct{}, readErr error) FeatureGateAccess { + return &hardcodedFeatureGateAccess{ + enabled: enabled, + disabled: disabled, + initialFeatureGatesObserved: initialFeatureGatesObserved, + readErr: readErr, + } +} + +func (c *hardcodedFeatureGateAccess) SetChangeHandler(featureGateChangeHandlerFn FeatureGateChangeHandlerFunc) { + // ignore +} + +func (c *hardcodedFeatureGateAccess) Run(ctx context.Context) { + // ignore +} + +func (c *hardcodedFeatureGateAccess) InitialFeatureGatesObserved() <-chan struct{} { + return c.initialFeatureGatesObserved +} + +func (c *hardcodedFeatureGateAccess) AreInitialFeatureGatesObserved() bool { + select { + case <-c.InitialFeatureGatesObserved(): + return true + default: + return false + } +} + +func (c *hardcodedFeatureGateAccess) CurrentFeatureGates() (FeatureGate, error) { + return NewFeatureGate(c.enabled, c.disabled), c.readErr +} + +// NewHardcodedFeatureGateAccessFromFeatureGate returns a FeatureGateAccess that is static and initialised from +// a populated FeatureGate status. +// If the desired version is missing, this will return an error. +func NewHardcodedFeatureGateAccessFromFeatureGate(featureGate *configv1.FeatureGate, desiredVersion string) (FeatureGateAccess, error) { + features, err := featuresFromFeatureGate(featureGate, desiredVersion) + if err != nil { + return nil, fmt.Errorf("unable to determine features: %w", err) + } + + return NewHardcodedFeatureGateAccess(features.Enabled, features.Disabled), nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/observe_featuregates.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/observe_featuregates.go new file mode 100644 index 00000000..0f2cb85f --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/observe_featuregates.go @@ -0,0 +1,118 @@ +package featuregates + +import ( + "fmt" + "reflect" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/library-go/pkg/operator/configobserver" + "github.com/openshift/library-go/pkg/operator/events" +) + +// NewObserveFeatureFlagsFunc produces a configobserver for feature gates. If non-nil, the featureWhitelist filters +// feature gates to a known subset (instead of everything). The featureBlacklist will stop certain features from making +// it through the list. The featureBlacklist should be empty, but for a brief time, some featuregates may need to skipped. +// @smarterclayton will live forever in shame for being the first to require this for "IPv6DualStack". +func NewObserveFeatureFlagsFunc(featureWhitelist sets.Set[configv1.FeatureGateName], featureBlacklist sets.Set[configv1.FeatureGateName], configPath []string, featureGateAccess FeatureGateAccess) configobserver.ObserveConfigFunc { + return (&featureFlags{ + allowAll: len(featureWhitelist) == 0, + featureWhitelist: featureWhitelist, + featureBlacklist: featureBlacklist, + configPath: configPath, + featureGateAccess: featureGateAccess, + }).ObserveFeatureFlags +} + +type featureFlags struct { + allowAll bool + featureWhitelist sets.Set[configv1.FeatureGateName] + // we add a forceDisableFeature list because we've now had bad featuregates break individual operators. Awesome. + featureBlacklist sets.Set[configv1.FeatureGateName] + configPath []string + featureGateAccess FeatureGateAccess +} + +// ObserveFeatureFlags fills in --feature-flags for the kube-apiserver +func (f *featureFlags) ObserveFeatureFlags(genericListers configobserver.Listers, recorder events.Recorder, existingConfig map[string]interface{}) (map[string]interface{}, []error) { + prunedExistingConfig := configobserver.Pruned(existingConfig, f.configPath) + + errs := []error{} + + if !f.featureGateAccess.AreInitialFeatureGatesObserved() { + // if we haven't observed featuregates yet, return the existing + return prunedExistingConfig, nil + } + + featureGates, err := f.featureGateAccess.CurrentFeatureGates() + if err != nil { + return prunedExistingConfig, append(errs, err) + } + observedConfig := map[string]interface{}{} + newConfigValue := f.getWhitelistedFeatureNames(featureGates) + + currentConfigValue, _, err := unstructured.NestedStringSlice(existingConfig, f.configPath...) + if err != nil { + errs = append(errs, err) + // keep going on read error from existing config + } + if !reflect.DeepEqual(currentConfigValue, newConfigValue) { + recorder.Eventf("ObserveFeatureFlagsUpdated", "Updated %v to %s", strings.Join(f.configPath, "."), strings.Join(newConfigValue, ",")) + } + + if err := unstructured.SetNestedStringSlice(observedConfig, newConfigValue, f.configPath...); err != nil { + recorder.Warningf("ObserveFeatureFlags", "Failed setting %v: %v", strings.Join(f.configPath, "."), err) + return prunedExistingConfig, append(errs, err) + } + + return configobserver.Pruned(observedConfig, f.configPath), errs +} + +func (f *featureFlags) getWhitelistedFeatureNames(featureGates FeatureGate) []string { + newConfigValue := []string{} + formatEnabledFunc := func(fs configv1.FeatureGateName) string { + return fmt.Sprintf("%v=true", fs) + } + formatDisabledFunc := func(fs configv1.FeatureGateName) string { + return fmt.Sprintf("%v=false", fs) + } + + for _, knownFeatureGate := range featureGates.KnownFeatures() { + if f.featureBlacklist.Has(knownFeatureGate) { + continue + } + // only add whitelisted feature flags + if !f.allowAll && !f.featureWhitelist.Has(knownFeatureGate) { + continue + } + + if featureGates.Enabled(knownFeatureGate) { + newConfigValue = append(newConfigValue, formatEnabledFunc(knownFeatureGate)) + } else { + newConfigValue = append(newConfigValue, formatDisabledFunc(knownFeatureGate)) + } + } + + return newConfigValue +} + +func StringsToFeatureGateNames(in []string) []configv1.FeatureGateName { + out := []configv1.FeatureGateName{} + for _, curr := range in { + out = append(out, configv1.FeatureGateName(curr)) + } + + return out +} + +func FeatureGateNamesToStrings(in []configv1.FeatureGateName) []string { + out := []string{} + for _, curr := range in { + out = append(out, string(curr)) + } + + return out +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/simple_featuregate_reader.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/simple_featuregate_reader.go new file mode 100644 index 00000000..4b2caccd --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/simple_featuregate_reader.go @@ -0,0 +1,318 @@ +package featuregates + +import ( + "context" + "fmt" + "os" + "reflect" + "sync" + "time" + + configv1 "github.com/openshift/api/config/v1" + + v1 "github.com/openshift/client-go/config/informers/externalversions/config/v1" + configlistersv1 "github.com/openshift/client-go/config/listers/config/v1" + "github.com/openshift/library-go/pkg/operator/events" + apierrors "k8s.io/apimachinery/pkg/api/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" +) + +type FeatureGateChangeHandlerFunc func(featureChange FeatureChange) + +// FeatureGateAccess is used to get a list of enabled and disabled featuregates. +// Create a new instance using NewFeatureGateAccess. +// To create one for unit testing, use NewHardcodedFeatureGateAccess. +type FeatureGateAccess interface { + // SetChangeHandler can only be called before Run. + // The default change handler will exit 0 when the set of featuregates changes. + // That is usually the easiest and simplest thing for an *operator* to do. + // This also discourages direct operand reading since all operands restarting simultaneously is bad. + // This function allows changing that default behavior to something else (perhaps a channel notification for + // all impacted controllers in an operator. + // I doubt this will be worth the effort in the majority of cases. + SetChangeHandler(featureGateChangeHandlerFn FeatureGateChangeHandlerFunc) + + // Run starts a go func that continously watches the set of featuregates enabled in the cluster. + Run(ctx context.Context) + // InitialFeatureGatesObserved returns a channel that is closed once the featuregates have + // been observed. Once closed, the CurrentFeatureGates method will return the current set of + // featuregates and will never return a non-nil error. + InitialFeatureGatesObserved() <-chan struct{} + // CurrentFeatureGates returns the list of enabled and disabled featuregates. + // It returns an error if the current set of featuregates is not known. + CurrentFeatureGates() (FeatureGate, error) + // AreInitialFeatureGatesObserved returns true if the initial featuregates have been observed. + AreInitialFeatureGatesObserved() bool +} + +type Features struct { + Enabled []configv1.FeatureGateName + Disabled []configv1.FeatureGateName +} + +type FeatureChange struct { + Previous *Features + New Features +} + +type defaultFeatureGateAccess struct { + desiredVersion string + missingVersionMarker string + clusterVersionLister configlistersv1.ClusterVersionLister + featureGateLister configlistersv1.FeatureGateLister + initialFeatureGatesObserved chan struct{} + + featureGateChangeHandlerFn FeatureGateChangeHandlerFunc + + lock sync.Mutex + started bool + initialFeatures Features + currentFeatures Features + + queue workqueue.RateLimitingInterface + eventRecorder events.Recorder +} + +// NewFeatureGateAccess returns a controller that keeps the list of enabled/disabled featuregates up to date. +// desiredVersion is the version of this operator that would be set on the clusteroperator.status.versions. +// missingVersionMarker is the stub version provided by the operator. If that is also the desired version, +// then the most either the desired clusterVersion or most recent version will be used. +// clusterVersionInformer is used when desiredVersion and missingVersionMarker are the same to derive the "best" version +// of featuregates to use. +// featureGateInformer is used to track changes to the featureGates once they are initially set. +// By default, when the enabled/disabled list of featuregates changes, os.Exit is called. This behavior can be +// overridden by calling SetChangeHandler to whatever you wish the behavior to be. +// A common construct is: +/* go +featureGateAccessor := NewFeatureGateAccess(args) +go featureGateAccessor.Run(ctx) + +select{ +case <- featureGateAccessor.InitialFeatureGatesObserved(): + featureGates, _ := featureGateAccessor.CurrentFeatureGates() + klog.Infof("FeatureGates initialized: knownFeatureGates=%v", featureGates.KnownFeatures()) +case <- time.After(1*time.Minute): + klog.Errorf("timed out waiting for FeatureGate detection") + return fmt.Errorf("timed out waiting for FeatureGate detection") +} + +// whatever other initialization you have to do, at this point you have FeatureGates to drive your behavior. +*/ +// That construct is easy. It is better to use the .spec.observedConfiguration construct common in library-go operators +// to avoid gating your general startup on FeatureGate determination, but if you haven't already got that mechanism +// this construct is easy. +func NewFeatureGateAccess( + desiredVersion, missingVersionMarker string, + clusterVersionInformer v1.ClusterVersionInformer, + featureGateInformer v1.FeatureGateInformer, + eventRecorder events.Recorder) FeatureGateAccess { + c := &defaultFeatureGateAccess{ + desiredVersion: desiredVersion, + missingVersionMarker: missingVersionMarker, + clusterVersionLister: clusterVersionInformer.Lister(), + featureGateLister: featureGateInformer.Lister(), + initialFeatureGatesObserved: make(chan struct{}), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "feature-gate-detector"), + eventRecorder: eventRecorder, + } + c.SetChangeHandler(ForceExit) + + // we aren't expecting many + clusterVersionInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.queue.Add("cluster") + }, + UpdateFunc: func(old, cur interface{}) { + c.queue.Add("cluster") + }, + DeleteFunc: func(uncast interface{}) { + c.queue.Add("cluster") + }, + }) + featureGateInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.queue.Add("cluster") + }, + UpdateFunc: func(old, cur interface{}) { + c.queue.Add("cluster") + }, + DeleteFunc: func(uncast interface{}) { + c.queue.Add("cluster") + }, + }) + + return c +} + +func ForceExit(featureChange FeatureChange) { + if featureChange.Previous != nil { + os.Exit(0) + } +} + +func (c *defaultFeatureGateAccess) SetChangeHandler(featureGateChangeHandlerFn FeatureGateChangeHandlerFunc) { + c.lock.Lock() + defer c.lock.Unlock() + + if c.started { + panic("programmer error, cannot update the change handler after starting") + } + c.featureGateChangeHandlerFn = featureGateChangeHandlerFn +} + +func (c *defaultFeatureGateAccess) Run(ctx context.Context) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + klog.Infof("Starting feature-gate-detector") + defer klog.Infof("Shutting down feature-gate-detector") + + go wait.UntilWithContext(ctx, c.runWorker, time.Second) + + <-ctx.Done() +} + +func (c *defaultFeatureGateAccess) syncHandler(ctx context.Context) error { + desiredVersion := c.desiredVersion + if c.missingVersionMarker == c.desiredVersion { + clusterVersion, err := c.clusterVersionLister.Get("version") + if apierrors.IsNotFound(err) { + return nil // we will be re-triggered when it is created + } + if err != nil { + return err + } + + desiredVersion = clusterVersion.Status.Desired.Version + if len(desiredVersion) == 0 && len(clusterVersion.Status.History) > 0 { + desiredVersion = clusterVersion.Status.History[0].Version + } + } + + featureGate, err := c.featureGateLister.Get("cluster") + if apierrors.IsNotFound(err) { + return nil // we will be re-triggered when it is created + } + if err != nil { + return err + } + + features, err := featuresFromFeatureGate(featureGate, desiredVersion) + if err != nil { + return fmt.Errorf("unable to determine features: %w", err) + } + + c.setFeatureGates(features) + + return nil +} + +func (c *defaultFeatureGateAccess) setFeatureGates(features Features) { + c.lock.Lock() + defer c.lock.Unlock() + + var previousFeatures *Features + if c.AreInitialFeatureGatesObserved() { + t := c.currentFeatures + previousFeatures = &t + } + + c.currentFeatures = features + + if !c.AreInitialFeatureGatesObserved() { + c.initialFeatures = features + close(c.initialFeatureGatesObserved) + c.eventRecorder.Eventf("FeatureGatesInitialized", "FeatureGates updated to %#v", c.currentFeatures) + } + + if previousFeatures == nil || !reflect.DeepEqual(*previousFeatures, c.currentFeatures) { + if previousFeatures != nil { + c.eventRecorder.Eventf("FeatureGatesModified", "FeatureGates updated to %#v", c.currentFeatures) + } + + c.featureGateChangeHandlerFn(FeatureChange{ + Previous: previousFeatures, + New: c.currentFeatures, + }) + } +} + +func (c *defaultFeatureGateAccess) InitialFeatureGatesObserved() <-chan struct{} { + return c.initialFeatureGatesObserved +} + +func (c *defaultFeatureGateAccess) AreInitialFeatureGatesObserved() bool { + select { + case <-c.InitialFeatureGatesObserved(): + return true + default: + return false + } +} + +func (c *defaultFeatureGateAccess) CurrentFeatureGates() (FeatureGate, error) { + c.lock.Lock() + defer c.lock.Unlock() + + if !c.AreInitialFeatureGatesObserved() { + return nil, fmt.Errorf("featureGates not yet observed") + } + retEnabled := make([]configv1.FeatureGateName, len(c.currentFeatures.Enabled)) + retDisabled := make([]configv1.FeatureGateName, len(c.currentFeatures.Disabled)) + copy(retEnabled, c.currentFeatures.Enabled) + copy(retDisabled, c.currentFeatures.Disabled) + + return NewFeatureGate(retEnabled, retDisabled), nil +} + +func (c *defaultFeatureGateAccess) runWorker(ctx context.Context) { + for c.processNextWorkItem(ctx) { + } +} + +func (c *defaultFeatureGateAccess) processNextWorkItem(ctx context.Context) bool { + dsKey, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(dsKey) + + err := c.syncHandler(ctx) + if err == nil { + c.queue.Forget(dsKey) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) + c.queue.AddRateLimited(dsKey) + + return true +} + +func featuresFromFeatureGate(featureGate *configv1.FeatureGate, desiredVersion string) (Features, error) { + found := false + features := Features{} + for _, featureGateValues := range featureGate.Status.FeatureGates { + if featureGateValues.Version != desiredVersion { + continue + } + found = true + for _, enabled := range featureGateValues.Enabled { + features.Enabled = append(features.Enabled, enabled.Name) + } + for _, disabled := range featureGateValues.Disabled { + features.Disabled = append(features.Disabled, disabled.Name) + } + break + } + + if !found { + return Features{}, fmt.Errorf("missing desired version %q in featuregates.config.openshift.io/cluster", desiredVersion) + } + + return features, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1039803a..38d6a3ae 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -288,6 +288,7 @@ github.com/openshift/library-go/pkg/crypto github.com/openshift/library-go/pkg/network github.com/openshift/library-go/pkg/operator/condition github.com/openshift/library-go/pkg/operator/configobserver +github.com/openshift/library-go/pkg/operator/configobserver/featuregates github.com/openshift/library-go/pkg/operator/configobserver/proxy github.com/openshift/library-go/pkg/operator/csi/credentialsrequestcontroller github.com/openshift/library-go/pkg/operator/csi/csiconfigobservercontroller